From 066630ed34ef07e7d48182ffed621eac3836bd30 Mon Sep 17 00:00:00 2001 From: Satya Patel <14907857+saddlepaddle@users.noreply.github.com> Date: Wed, 18 Nov 2026 21:58:47 +0000 Subject: [PATCH 1/8] feat(cloud-workspace): own workspace retention with a daily reaper New workspace boxes no longer expire their snapshots on Vercel's clock. A daily QStash job (reap-retention) fails a workspace asleep and unused for 30 days as snapshot_expired, the state the 410 path gives, retires its box name in the same update, and queues a retried box teardown. In production it also tears down tagged boxes no row or unarchived environment names, 7 days old, one delete a second. Counts go to logs, Sentry logs and a cron monitor. The teardown is box-only with #9847's route and payload, so its drive version replaces it. A dry-run-by-default backfill sets snapshotExpiration 0 on existing boxes (not run). --- .../workflows/publish-cloud-workspace-job.yml | 1 + .../jobs/reap-retention/route.ts | 31 ++ .../api/cloud-workspaces/teardown/route.ts | 32 ++ docs/cloud-sandbox-mismatches.md | 23 ++ packages/sandbox/package.json | 1 + .../src/backfill-snapshot-expiration.test.ts | 39 +++ .../src/backfill-snapshot-expiration.ts | 105 ++++++ packages/trpc/package.json | 8 + packages/trpc/src/lib/sandbox/index.ts | 2 + packages/trpc/src/lib/sandbox/vercel.test.ts | 29 ++ packages/trpc/src/lib/sandbox/vercel.ts | 34 +- .../router/cloud-workspace/retention.test.ts | 196 +++++++++++ .../src/router/cloud-workspace/retention.ts | 314 ++++++++++++++++++ .../router/cloud-workspace/teardown-guard.ts | 38 +++ .../router/cloud-workspace/teardown.test.ts | 73 ++++ .../src/router/cloud-workspace/teardown.ts | 82 +++++ .../src/router/cloud-workspace/transition.ts | 24 +- ...8-1500-cloud-workspace-retention-reaper.md | 83 +++++ 18 files changed, 1104 insertions(+), 11 deletions(-) create mode 100644 apps/api/src/app/api/cloud-workspaces/jobs/reap-retention/route.ts create mode 100644 apps/api/src/app/api/cloud-workspaces/teardown/route.ts create mode 100644 packages/sandbox/src/backfill-snapshot-expiration.test.ts create mode 100644 packages/sandbox/src/backfill-snapshot-expiration.ts create mode 100644 packages/trpc/src/router/cloud-workspace/retention.test.ts create mode 100644 packages/trpc/src/router/cloud-workspace/retention.ts create mode 100644 packages/trpc/src/router/cloud-workspace/teardown-guard.ts create mode 100644 packages/trpc/src/router/cloud-workspace/teardown.test.ts create mode 100644 packages/trpc/src/router/cloud-workspace/teardown.ts create mode 100644 plans/done/20261118-1500-cloud-workspace-retention-reaper.md diff --git a/.github/workflows/publish-cloud-workspace-job.yml b/.github/workflows/publish-cloud-workspace-job.yml index 51f02cea3eb..20dacb277e2 100644 --- a/.github/workflows/publish-cloud-workspace-job.yml +++ b/.github/workflows/publish-cloud-workspace-job.yml @@ -9,6 +9,7 @@ on: options: - reconcile-usage - renew-chatgpt-sign-ins + - reap-retention default: reconcile-usage jobs: diff --git a/apps/api/src/app/api/cloud-workspaces/jobs/reap-retention/route.ts b/apps/api/src/app/api/cloud-workspaces/jobs/reap-retention/route.ts new file mode 100644 index 00000000000..f108794b8cc --- /dev/null +++ b/apps/api/src/app/api/cloud-workspaces/jobs/reap-retention/route.ts @@ -0,0 +1,31 @@ +import * as Sentry from "@sentry/nextjs"; +import { + RETENTION_JOB_PATH, + reapCloudWorkspaceRetention, +} from "@superset/trpc/cloud-workspace-retention"; +import { verifyQstashRequest } from "@/lib/verifyQstash"; + +export const dynamic = "force-dynamic"; +export const maxDuration = 300; + +/** + * Run by a QStash schedule daily at 06:00 UTC. That schedule lives in the + * Upstash console, not in this repo. + */ +export async function POST(request: Request): Promise { + const body = await request.text(); + const rejected = await verifyQstashRequest(request, body, RETENTION_JOB_PATH); + if (rejected) return rejected; + + const outcome = await Sentry.withMonitor( + "reap-retention", + () => + reapCloudWorkspaceRetention({ + production: process.env.VERCEL_ENV === "production", + }), + { schedule: { type: "crontab", value: "0 6 * * *" } }, + ); + // Turbopack builds do not wrap route handlers, so nothing else flushes the closing check-in. + await Sentry.flush(2_000).catch(() => false); + return Response.json({ ok: true, ...outcome }); +} diff --git a/apps/api/src/app/api/cloud-workspaces/teardown/route.ts b/apps/api/src/app/api/cloud-workspaces/teardown/route.ts new file mode 100644 index 00000000000..46ec2aa7a4f --- /dev/null +++ b/apps/api/src/app/api/cloud-workspaces/teardown/route.ts @@ -0,0 +1,32 @@ +import { + TEARDOWN_JOB_PATH, + teardownSandbox, +} from "@superset/trpc/cloud-workspace-teardown"; +import { z } from "zod"; +import { verifyQstashRequest } from "@/lib/verifyQstash"; + +export const dynamic = "force-dynamic"; +export const maxDuration = 60; + +const payloadSchema = z + .object({ + providerSandboxId: z.string().min(1), + driveName: z.null(), + attempt: z.number().int().min(0), + }) + .strict(); + +/** Stops a box, then deletes it with its snapshots; requeues itself on failure. */ +export async function POST(request: Request): Promise { + const body = await request.text(); + const rejected = await verifyQstashRequest(request, body, TEARDOWN_JOB_PATH); + if (rejected) return rejected; + + const parsed = payloadSchema.safeParse(JSON.parse(body)); + if (!parsed.success) { + console.error("[cloud-workspaces/teardown] invalid payload", parsed.error); + return Response.json({ error: "Invalid payload" }, { status: 400 }); + } + const outcome = await teardownSandbox(parsed.data); + return Response.json({ ok: true, outcome }); +} diff --git a/docs/cloud-sandbox-mismatches.md b/docs/cloud-sandbox-mismatches.md index 803c699ddca..29ac1a7783f 100644 --- a/docs/cloud-sandbox-mismatches.md +++ b/docs/cloud-sandbox-mismatches.md @@ -783,6 +783,29 @@ sandbox is gone or unresumable is marked `failed` by the next `access`, so it gets the failed screen and a Remove button rather than a sidebar entry that never opens. +**Vercel's snapshot expiry was silent, so we own retention now.** The app +assumed a stopped box keeps its disk until someone deletes it. In fact Vercel +expired a workspace box's snapshot 30 days after its last use. Nothing told us: +the row learned only from the 410 that its next wake, agent turn or automation +dispatch got. A workspace nobody opened again stayed `asleep` over a box that +could not resume, and the box itself stayed in Vercel. New workspace boxes are +now created with `snapshotExpiration: 0`. The daily `reap-retention` job +(`router/cloud-workspace/retention.ts`) does three things. It moves a workspace +that is asleep and unused for 30 days to `failed`/`snapshot_expired`, the same +state as the 410. It gives the row a new box name in that same update. Then it +queues `teardownSandbox`, which deletes the box only if no row names it. "Unused" +means no boot started or stopped since then, with `sandbox_expires_at` and +`created_at` older too, so a wake in flight (its boot row is written before +Vercel resumes) keeps the box. In production only, the same job tears down +tagged boxes that no row and no unarchived environment names, once they are +7 days old: every deployment shares one Vercel project, so a dev or preview +database would see production's boxes as orphans. Boxes created before this +change keep a 30-day expiry until `bun run backfill-snapshot-expiration +--apply` (`packages/sandbox`) is run. That run changes only snapshots taken +afterwards, so until a box's next stop the 410 path stays its fallback. +**Open:** drives are not reaped yet. No row names a workspace drive on main, +so that half waits for the drive columns (#9847). + **Any call on a `Sandbox` object resumes a stopped session.** The SDK wraps `runCommand`, `writeFiles`, `extendTimeout` and the rest in `withResume`: on a "stopped" answer it resumes and retries. A box read `running` that hits its diff --git a/packages/sandbox/package.json b/packages/sandbox/package.json index e00d676269f..f03e0f590c9 100644 --- a/packages/sandbox/package.json +++ b/packages/sandbox/package.json @@ -9,6 +9,7 @@ "pin": "bun run src/pin.ts", "assets": "bun run src/assets/produce.ts", "inventory": "bun run src/inventory.ts", + "backfill-snapshot-expiration": "bun run src/backfill-snapshot-expiration.ts", "release": "bun run src/environments/release.ts", "agent-tools": "bun run src/agent-tools.ts", "record-bundle": "bun run src/environments/record-bundle.ts", diff --git a/packages/sandbox/src/backfill-snapshot-expiration.test.ts b/packages/sandbox/src/backfill-snapshot-expiration.test.ts new file mode 100644 index 00000000000..f7c072aa666 --- /dev/null +++ b/packages/sandbox/src/backfill-snapshot-expiration.test.ts @@ -0,0 +1,39 @@ +import { expect, test } from "bun:test"; +import { backfillSnapshotExpiration } from "./backfill-snapshot-expiration"; + +const boxes = [ + { name: "ws-default", snapshotExpiration: undefined }, + { name: "ws-30d", snapshotExpiration: 30 * 24 * 60 * 60 * 1000 }, + { name: "ws-done", snapshotExpiration: 0 }, +]; + +test("changes only boxes that still expire, paced, and only with --apply", async () => { + const updated: string[] = []; + const slept: number[] = []; + const host = { + listWorkspaceBoxes: async () => boxes, + neverExpire: async (name: string) => { + updated.push(name); + }, + sleep: async (ms: number) => { + slept.push(ms); + }, + }; + const dry = await backfillSnapshotExpiration({ + host, + apply: false, + perSecond: 2, + log: () => {}, + }); + expect(dry).toMatchObject({ pending: 2, updated: 0 }); + expect(updated).toEqual([]); + + await backfillSnapshotExpiration({ + host, + apply: true, + perSecond: 2, + log: () => {}, + }); + expect(updated).toEqual(["ws-default", "ws-30d"]); + expect(slept).toEqual([500]); +}); diff --git a/packages/sandbox/src/backfill-snapshot-expiration.ts b/packages/sandbox/src/backfill-snapshot-expiration.ts new file mode 100644 index 00000000000..7e5449292d2 --- /dev/null +++ b/packages/sandbox/src/backfill-snapshot-expiration.ts @@ -0,0 +1,105 @@ +/** + * Sets `snapshotExpiration: 0` on workspace boxes made before new boxes got + * it, so Vercel stops expiring their snapshots and the retention reaper + * decides instead. It changes the default for snapshots taken from then on: + * a box's current snapshot keeps the expiry it was taken with, so a box only + * stops expiring after its next stop. Wakes nothing. Dry run by default. + * + * bun run backfill-snapshot-expiration lists what it would change + * bun run backfill-snapshot-expiration --apply changes them, 2 a second + * + * Needs VERCEL_SANDBOX_TOKEN, VERCEL_SANDBOX_TEAM_ID and VERCEL_SANDBOX_PROJECT_ID. + */ +import { Sandbox } from "@vercel/sandbox"; +import { missingCredentials } from "./inventory"; + +export interface BackfillBox { + name: string; + snapshotExpiration: number | undefined; +} + +export interface BackfillHost { + listWorkspaceBoxes(): Promise; + neverExpire(name: string): Promise; + sleep(ms: number): Promise; +} + +export async function backfillSnapshotExpiration(args: { + host: BackfillHost; + apply: boolean; + perSecond: number; + log: (line: string) => void; +}): Promise<{ + listed: number; + pending: number; + updated: number; + failed: number; +}> { + const boxes = await args.host.listWorkspaceBoxes(); + const pending = boxes.filter((box) => box.snapshotExpiration !== 0); + args.log(`${boxes.length} workspace boxes, ${pending.length} still expire`); + let updated = 0; + let failed = 0; + if (!args.apply) { + for (const box of pending) { + args.log( + `would set ${box.name} (now ${box.snapshotExpiration ?? "default"})`, + ); + } + if (pending.length) args.log("dry run: pass --apply to change them"); + return { listed: boxes.length, pending: pending.length, updated, failed }; + } + for (const [index, box] of pending.entries()) { + if (index > 0) await args.host.sleep(1000 / args.perSecond); + try { + await args.host.neverExpire(box.name); + updated++; + } catch (error) { + failed++; + args.log(`::warning::could not update ${box.name}: ${error}`); + } + } + args.log(`updated ${updated}, failed ${failed}`); + return { listed: boxes.length, pending: pending.length, updated, failed }; +} + +if (import.meta.main) { + const missing = missingCredentials(process.env); + if (missing.length > 0) { + console.error( + `backfill needs real Vercel sandbox credentials; missing or placeholder: ${missing.join(", ")}`, + ); + process.exit(64); + } + const credentials = { + token: process.env.VERCEL_SANDBOX_TOKEN ?? "", + teamId: process.env.VERCEL_SANDBOX_TEAM_ID ?? "", + projectId: process.env.VERCEL_SANDBOX_PROJECT_ID ?? "", + }; + const { failed } = await backfillSnapshotExpiration({ + host: { + async listWorkspaceBoxes() { + const boxes: BackfillBox[] = []; + for await (const box of await Sandbox.list({ + ...credentials, + tags: { kind: "workspace" }, + })) { + boxes.push({ + name: box.name, + snapshotExpiration: box.snapshotExpiration, + }); + } + return boxes; + }, + async neverExpire(name) { + const box = await Sandbox.get({ ...credentials, name, resume: false }); + await box.update({ snapshotExpiration: 0 }); + }, + sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)), + }, + apply: process.argv.includes("--apply"), + perSecond: 2, + log: console.log, + }); + process.exit(failed ? 1 : 0); +} diff --git a/packages/trpc/package.json b/packages/trpc/package.json index c444ba70db4..a71a18c6c48 100644 --- a/packages/trpc/package.json +++ b/packages/trpc/package.json @@ -136,6 +136,14 @@ "types": "./src/router/cloud-workspace/reap.ts", "default": "./src/router/cloud-workspace/reap.ts" }, + "./cloud-workspace-retention": { + "types": "./src/router/cloud-workspace/retention.ts", + "default": "./src/router/cloud-workspace/retention.ts" + }, + "./cloud-workspace-teardown": { + "types": "./src/router/cloud-workspace/teardown.ts", + "default": "./src/router/cloud-workspace/teardown.ts" + }, "./automation-dispatch": { "types": "./src/router/automation/dispatch.ts", "default": "./src/router/automation/dispatch.ts" diff --git a/packages/trpc/src/lib/sandbox/index.ts b/packages/trpc/src/lib/sandbox/index.ts index ec376ff4943..2f6de7e6908 100644 --- a/packages/trpc/src/lib/sandbox/index.ts +++ b/packages/trpc/src/lib/sandbox/index.ts @@ -61,6 +61,7 @@ export { HOST_SERVICE_PORT, listActiveWorkspaceSandboxes, listSandboxSessions, + listTaggedSandboxes, MEMORY_MB_PER_VCPU, promoteSandboxToEnvironment, provisionSandbox, @@ -75,6 +76,7 @@ export { stopAndSnapshot, stopSandbox, stripWorkspaceIdentity, + type TaggedSandbox, topUpSandboxSession, waitForStopSnapshot, wakeSandbox, diff --git a/packages/trpc/src/lib/sandbox/vercel.test.ts b/packages/trpc/src/lib/sandbox/vercel.test.ts index 2141b68005c..e7fe57e7ab4 100644 --- a/packages/trpc/src/lib/sandbox/vercel.test.ts +++ b/packages/trpc/src/lib/sandbox/vercel.test.ts @@ -17,6 +17,7 @@ import { listActiveWorkspaceSandboxes, listSandboxSessions, promoteSandboxToEnvironment, + provisionSandbox, type SandboxClaim, SandboxUnavailableError, SESSION_TIMEOUT_MS, @@ -307,6 +308,34 @@ describe("a caller that is not a wake leaves a stopped box stopped", () => { }); }); +describe("provisionSandbox", () => { + let created: { snapshotExpiration?: number } | undefined; + stub(Sandbox, { + get: async () => { + throw new APIError(new Response(null, { status: 404 })); + }, + create: async (params: { snapshotExpiration?: number }) => { + created = params; + throw new Error("created"); + }, + }); + + test("a workspace box's snapshots never expire on Vercel's clock", async () => { + const provision = provisionSandbox({ + name: "ws-new", + environment: { sourceKind: "image", sourceRef: "img", region: "iad1" }, + claim: { + identity: {}, + hostSecret: "secret", + networkPolicy: "allow-all", + vcpus: 2, + } as unknown as SandboxClaim, + }); + await expect(provision).rejects.toThrow("created"); + expect(created?.snapshotExpiration).toBe(0); + }); +}); + describe("promoteSandboxToEnvironment", () => { let fake = fakeBox("running"); let sourceSnapshot: string | undefined; diff --git a/packages/trpc/src/lib/sandbox/vercel.ts b/packages/trpc/src/lib/sandbox/vercel.ts index b79ff3809e9..fbed4185c2f 100644 --- a/packages/trpc/src/lib/sandbox/vercel.ts +++ b/packages/trpc/src/lib/sandbox/vercel.ts @@ -46,7 +46,6 @@ export const SESSION_TIMEOUT_MS = 60 * 60 * 1000; /** Above the 10-minute client keepalive, so an open workspace is always extended in time. */ const EXTEND_BELOW_MS = 30 * 60 * 1000; const MIN_TOP_UP_MS = 60 * 1000; -const WORKSPACE_SNAPSHOT_EXPIRATION_MS = 30 * 24 * 60 * 60 * 1000; /** 2 GB of memory per vCPU; disk is 64 GB regardless. */ export const MEMORY_MB_PER_VCPU = 2048; const GOLDEN_SESSION_TIMEOUT_MS = 30 * 60 * 1000; @@ -304,7 +303,6 @@ export async function provisionSandbox(args: { name: string; environment: SandboxEnvironment; claim: SandboxClaim; - /** A golden under construction keeps its snapshots until its row goes. */ kind?: "workspace" | "environment"; }): Promise<{ providerSandboxId: string; @@ -322,8 +320,8 @@ export async function provisionSandbox(args: { env: {}, networkPolicy: args.claim.networkPolicy, persistent: true, - snapshotExpiration: - kind === "environment" ? 0 : WORKSPACE_SNAPSHOT_EXPIRATION_MS, + // Vercel never expires a snapshot: the retention reaper decides when a box goes. + snapshotExpiration: 0, keepLastSnapshots: { count: 1 }, tags: sandboxTags(kind, args.claim.identity), }; @@ -989,6 +987,34 @@ export async function listActiveWorkspaceSandboxes( return found; } +export interface TaggedSandbox { + name: string; + createdAt: number; + statusUpdatedAt: number; + status: string; + snapshotExpiration: number | undefined; +} + +/** Every box we tagged with this kind, whatever its age. */ +export async function listTaggedSandboxes( + kind: "workspace" | "environment", +): Promise { + const found: TaggedSandbox[] = []; + for await (const sandbox of await Sandbox.list({ + ...credentials(), + tags: { kind }, + })) { + found.push({ + name: sandbox.name, + createdAt: sandbox.createdAt, + statusUpdatedAt: sandbox.statusUpdatedAt ?? sandbox.updatedAt, + status: sandbox.status, + snapshotExpiration: sandbox.snapshotExpiration, + }); + } + return found; +} + /** Best-effort: a sandbox already gone is the state we wanted. */ export async function deleteSandbox(providerSandboxId: string): Promise { const sandbox = await getSandbox(providerSandboxId); diff --git a/packages/trpc/src/router/cloud-workspace/retention.test.ts b/packages/trpc/src/router/cloud-workspace/retention.test.ts new file mode 100644 index 00000000000..00170b09d4f --- /dev/null +++ b/packages/trpc/src/router/cloud-workspace/retention.test.ts @@ -0,0 +1,196 @@ +import { describe, expect, test } from "bun:test"; +import { stub } from "../../../test/stub"; +import type { TaggedSandbox } from "../../lib/sandbox"; +import { + ORPHAN_BATCH, + RETENTION_IDLE_MS, + type RetentionDeps, + reapCloudWorkspaceRetention, + retireIdleWorkspace, +} from "./retention"; +import * as transition from "./transition"; + +const NOW = Date.parse("2026-11-18T06:00:00Z"); +const DAY = 24 * 60 * 60 * 1000; + +const idleRow = (n: number) => ({ + id: `ws_${n}`, + organizationId: "org_1", + providerSandboxId: `ws-box${n}`, +}); + +const box = (name: string, overrides: Partial = {}) => ({ + name, + createdAt: NOW - 60 * DAY, + statusUpdatedAt: NOW - 40 * DAY, + status: "stopped", + snapshotExpiration: 0, + ...overrides, +}); + +function fakeDeps(overrides: Partial = {}) { + const queued: Array<[string, number]> = []; + const retired: string[] = []; + const deps: RetentionDeps = { + findIdleWorkspaces: async () => [], + retireIdleWorkspace: async (row) => { + retired.push(row.id); + return true; + }, + boxIsRunning: async () => false, + listTaggedSandboxes: async () => [], + namedSandboxes: async () => new Set(), + queueTeardown: async (name, delaySeconds) => { + queued.push([name, delaySeconds]); + }, + ...overrides, + }; + return { deps, queued, retired }; +} + +describe("reapCloudWorkspaceRetention", () => { + test("retires each idle workspace and queues its box's teardown a second apart", async () => { + let cutoff: Date | undefined; + const { deps, queued, retired } = fakeDeps({ + findIdleWorkspaces: async (at) => { + cutoff = at; + return [idleRow(1), idleRow(2)]; + }, + }); + const run = await reapCloudWorkspaceRetention({ + now: NOW, + production: false, + deps, + }); + expect(cutoff?.getTime()).toBe(NOW - RETENTION_IDLE_MS); + expect(retired).toEqual(["ws_1", "ws_2"]); + expect(queued).toEqual([ + ["ws-box1", 0], + ["ws-box2", 1], + ]); + expect(run.idle).toMatchObject({ found: 2, retired: 2 }); + }); + + test("a box that runs a session now is skipped, not retired", async () => { + const { deps, queued, retired } = fakeDeps({ + findIdleWorkspaces: async () => [idleRow(1)], + boxIsRunning: async () => true, + }); + const run = await reapCloudWorkspaceRetention({ + now: NOW, + production: false, + deps, + }); + expect(retired).toEqual([]); + expect(queued).toEqual([]); + expect(run.idle.skippedRunning).toBe(1); + }); + + test("a row woken or changed after it was read keeps its box", async () => { + const { deps, queued } = fakeDeps({ + findIdleWorkspaces: async () => [idleRow(1)], + retireIdleWorkspace: async () => false, + }); + const run = await reapCloudWorkspaceRetention({ + now: NOW, + production: false, + deps, + }); + expect(queued).toEqual([]); + expect(run.idle.skippedChanged).toBe(1); + }); + + test("queues only old, settled boxes no row names, and not one it just retired", async () => { + const { deps, queued } = fakeDeps({ + findIdleWorkspaces: async () => [idleRow(1)], + listTaggedSandboxes: async (kind) => + kind === "workspace" + ? [ + box("ws-box1"), + box("ws-orphan"), + box("ws-named"), + box("ws-young", { createdAt: NOW - DAY }), + box("ws-recent", { statusUpdatedAt: NOW - DAY }), + box("ws-running", { status: "running" }), + ] + : [box("env-orphan")], + namedSandboxes: async () => new Set(["ws-named"]), + }); + const run = await reapCloudWorkspaceRetention({ + now: NOW, + production: true, + deps, + }); + expect(queued).toEqual([ + ["ws-box1", 0], + ["ws-orphan", 1], + ["env-orphan", 2], + ]); + expect(run.orphans).toEqual({ + listed: 7, + orphaned: 2, + queued: 2, + deferred: 0, + failed: 0, + }); + }); + + test("outside production it never looks for orphans", async () => { + let listed = false; + const { deps } = fakeDeps({ + listTaggedSandboxes: async () => { + listed = true; + return [box("ws-orphan")]; + }, + }); + const run = await reapCloudWorkspaceRetention({ + now: NOW, + production: false, + deps, + }); + expect(listed).toBe(false); + expect(run.orphans).toEqual({ skipped: "not production" }); + }); + + test("queues at most a batch of orphans a run and counts the rest", async () => { + const many = Array.from({ length: ORPHAN_BATCH + 3 }, (_, i) => + box(`ws-orphan${i}`), + ); + const { deps, queued } = fakeDeps({ + listTaggedSandboxes: async (kind) => (kind === "workspace" ? many : []), + }); + const run = await reapCloudWorkspaceRetention({ + now: NOW, + production: true, + deps, + }); + expect(queued).toHaveLength(ORPHAN_BATCH); + expect(run.orphans).toMatchObject({ queued: ORPHAN_BATCH, deferred: 3 }); + }); +}); + +describe("retireIdleWorkspace", () => { + let failed: Parameters[0] | undefined; + stub(transition, { + failCloudWorkspace: async ( + args: Parameters[0], + ) => { + failed = args; + return true; + }, + }); + + test("fails the row as an expired snapshot and gives it a new box name in the same update", async () => { + await retireIdleWorkspace(idleRow(1), new Date(NOW - RETENTION_IDLE_MS)); + expect(failed).toMatchObject({ + scope: { cloudWorkspaceId: "ws_1", organizationId: "org_1" }, + from: ["asleep"], + reason: "snapshot_expired", + expected: true, + set: { sandboxUrl: null }, + }); + expect(failed?.set?.providerSandboxId).toMatch(/^ws-ws_1[0-9a-f]{8}$/); + expect(failed?.set?.providerSandboxId).not.toBe("ws-box1"); + expect(failed?.where).toBeDefined(); + }); +}); diff --git a/packages/trpc/src/router/cloud-workspace/retention.ts b/packages/trpc/src/router/cloud-workspace/retention.ts new file mode 100644 index 00000000000..52711576091 --- /dev/null +++ b/packages/trpc/src/router/cloud-workspace/retention.ts @@ -0,0 +1,314 @@ +import * as Sentry from "@sentry/core"; +import { db } from "@superset/db/client"; +import { cloudWorkspaceBoots, cloudWorkspaces } from "@superset/db/schema"; +import { and, asc, eq, isNull, lt, or, type SQL, sql } from "drizzle-orm"; +import { + describeSandbox, + listTaggedSandboxes, + SandboxUnavailableError, + type TaggedSandbox, +} from "../../lib/sandbox"; +import { nextSandboxNameFor } from "./provision"; +import { queueTeardown } from "./teardown"; +import { namedSandboxes } from "./teardown-guard"; +import { failCloudWorkspace } from "./transition"; + +export const RETENTION_JOB_PATH = "/api/cloud-workspaces/jobs/reap-retention"; + +const DAY_MS = 24 * 60 * 60 * 1000; +/** What Vercel's snapshot expiry gave customers before we owned it. */ +export const RETENTION_IDLE_MS = 30 * DAY_MS; +/** Longer than a release or a promote takes to write the name of a golden it built. */ +export const ORPHAN_MIN_AGE_MS = 7 * DAY_MS; +export const REAP_BATCH = 50; +export const ORPHAN_BATCH = 50; +/** One delete a second: far under Vercel's 20 deletions a second per team. */ +const TEARDOWN_SPACING_SECONDS = 1; +const NAME_LOOKUP_CHUNK = 500; +const SETTLED_STATUSES = new Set(["stopped", "failed", "aborted"]); + +export interface IdleWorkspace { + id: string; + organizationId: string; + providerSandboxId: string; +} + +/** + * Nothing used the workspace since `cutoff`. Boots count because a wake + * writes its boot before Vercel resumes the box, so one in flight keeps the + * row. `sandbox_expires_at` and `created_at` count too: when they disagree + * with the boots, the later one wins and the box is kept longer. + */ +export function idleSince(cutoff: Date): SQL { + const at = sql`${cutoff.toISOString()}::timestamptz`; + return and( + lt(cloudWorkspaces.createdAt, cutoff), + or( + isNull(cloudWorkspaces.sandboxExpiresAt), + lt(cloudWorkspaces.sandboxExpiresAt, cutoff), + ), + sql`not exists (select 1 from ${cloudWorkspaceBoots} where ${cloudWorkspaceBoots.cloudWorkspaceId} = ${cloudWorkspaces.id} and (${cloudWorkspaceBoots.stoppedAt} is null or ${cloudWorkspaceBoots.stoppedAt} >= ${at} or ${cloudWorkspaceBoots.startedAt} >= ${at}))`, + ) as SQL; +} + +function stillIdleAndAsleep(cutoff: Date): SQL { + return and( + eq(cloudWorkspaces.provider, "vercel"), + isNull(cloudWorkspaces.deletedAt), + idleSince(cutoff), + ) as SQL; +} + +async function findIdleWorkspaces( + cutoff: Date, + limit: number, +): Promise { + return db + .select({ + id: cloudWorkspaces.id, + organizationId: cloudWorkspaces.organizationId, + providerSandboxId: cloudWorkspaces.providerSandboxId, + }) + .from(cloudWorkspaces) + .where( + and(eq(cloudWorkspaces.status, "asleep"), stillIdleAndAsleep(cutoff)), + ) + .orderBy(asc(cloudWorkspaces.createdAt)) + .limit(limit); +} + +/** + * The 410 path's terminal state, and the box's name taken off the row in the + * same update, so nothing can wake the box being deleted. False when the row + * changed since it was read: woken, archived, or given another box. + */ +export function retireIdleWorkspace( + row: IdleWorkspace, + cutoff: Date, +): Promise { + return failCloudWorkspace({ + scope: { cloudWorkspaceId: row.id, organizationId: row.organizationId }, + from: ["asleep"], + where: and( + eq(cloudWorkspaces.providerSandboxId, row.providerSandboxId), + stillIdleAndAsleep(cutoff), + ), + set: { + sandboxUrl: null, + providerSandboxId: nextSandboxNameFor(row.id), + }, + reason: "snapshot_expired", + error: new Error(`idle since before ${cutoff.toISOString()}`), + expected: true, + }); +} + +/** True when the box runs a session now; false when it is stopped or gone. */ +async function boxIsRunning(providerSandboxId: string): Promise { + try { + return (await describeSandbox(providerSandboxId)).running; + } catch (error) { + if (error instanceof SandboxUnavailableError) return false; + throw error; + } +} + +async function namedAmong(names: string[]): Promise> { + const named = new Set(); + for (let i = 0; i < names.length; i += NAME_LOOKUP_CHUNK) { + for (const name of await namedSandboxes( + names.slice(i, i + NAME_LOOKUP_CHUNK), + )) { + named.add(name); + } + } + return named; +} + +export interface RetentionDeps { + findIdleWorkspaces: typeof findIdleWorkspaces; + retireIdleWorkspace: typeof retireIdleWorkspace; + boxIsRunning: typeof boxIsRunning; + listTaggedSandboxes: typeof listTaggedSandboxes; + namedSandboxes: typeof namedAmong; + queueTeardown: ( + providerSandboxId: string, + delaySeconds: number, + ) => Promise; +} + +const defaultDeps: RetentionDeps = { + findIdleWorkspaces, + retireIdleWorkspace, + boxIsRunning, + listTaggedSandboxes, + namedSandboxes: namedAmong, + queueTeardown: (providerSandboxId, delaySeconds) => + queueTeardown( + { providerSandboxId, driveName: null, attempt: 0 }, + delaySeconds, + ), +}; + +export interface RetentionRun { + idle: { + found: number; + retired: number; + skippedRunning: number; + skippedChanged: number; + failed: number; + }; + orphans: + | { + listed: number; + orphaned: number; + queued: number; + deferred: number; + failed: number; + } + | { skipped: "not production" }; +} + +/** + * The one place a workspace box is deleted for age. Daily: a workspace asleep + * and unused for `RETENTION_IDLE_MS` fails as `snapshot_expired`, exactly as a + * wake that met Vercel's expiry did, and its box is torn down. Then, in + * production only, boxes we tagged that no row names are torn down. Every + * deployment shares the Vercel project, so another database would see + * production's boxes as orphans. + */ +export async function reapCloudWorkspaceRetention({ + now = Date.now(), + production, + deps = defaultDeps, +}: { + now?: number; + production: boolean; + deps?: RetentionDeps; +}): Promise { + let slot = 0; + const queue = (name: string) => + deps.queueTeardown(name, slot++ * TEARDOWN_SPACING_SECONDS); + const retiredNames = new Set(); + const idle = await reapIdle({ now, deps, queue, retiredNames }); + const orphans = production + ? await reapOrphans({ now, deps, queue, retiredNames }) + : ({ skipped: "not production" } as const); + const run = { idle, orphans }; + console.log("[cloud-workspace] retention run", JSON.stringify(run)); + Sentry.logger.info("retention.run", flatten(run)); + if (idle.failed || ("failed" in orphans && orphans.failed)) { + Sentry.captureMessage("cloud workspace retention run had failures", { + level: "warning", + extra: { ...run }, + }); + } + return run; +} + +async function reapIdle({ + now, + deps, + queue, + retiredNames, +}: { + now: number; + deps: RetentionDeps; + queue: (name: string) => Promise; + retiredNames: Set; +}): Promise { + const cutoff = new Date(now - RETENTION_IDLE_MS); + const rows = await deps.findIdleWorkspaces(cutoff, REAP_BATCH); + const counts = { + found: rows.length, + retired: 0, + skippedRunning: 0, + skippedChanged: 0, + failed: 0, + }; + for (const row of rows) { + try { + if (await deps.boxIsRunning(row.providerSandboxId)) { + counts.skippedRunning++; + continue; + } + if (!(await deps.retireIdleWorkspace(row, cutoff))) { + counts.skippedChanged++; + continue; + } + counts.retired++; + retiredNames.add(row.providerSandboxId); + // A retired box whose teardown is not queued is an orphan: a later run queues it. + await queue(row.providerSandboxId); + } catch (error) { + counts.failed++; + Sentry.captureException(error, { + tags: { "cloud_workspace.id": row.id }, + }); + } + } + return counts; +} + +function isOldAndSettled(box: TaggedSandbox, now: number): boolean { + return ( + SETTLED_STATUSES.has(box.status) && + now - box.createdAt > ORPHAN_MIN_AGE_MS && + now - box.statusUpdatedAt > ORPHAN_MIN_AGE_MS + ); +} + +async function reapOrphans({ + now, + deps, + queue, + retiredNames, +}: { + now: number; + deps: RetentionDeps; + queue: (name: string) => Promise; + retiredNames: Set; +}): Promise> { + const boxes = [ + ...(await deps.listTaggedSandboxes("workspace")), + ...(await deps.listTaggedSandboxes("environment")), + ]; + const candidates = boxes + .filter((box) => isOldAndSettled(box, now) && !retiredNames.has(box.name)) + .map((box) => box.name); + const named = await deps.namedSandboxes(candidates); + const orphaned = candidates.filter((name) => !named.has(name)); + const batch = orphaned.slice(0, ORPHAN_BATCH); + let queued = 0; + let failed = 0; + for (const name of batch) { + try { + await queue(name); + queued++; + } catch (error) { + failed++; + Sentry.captureException(error, { extra: { providerSandboxId: name } }); + } + } + return { + listed: boxes.length, + orphaned: orphaned.length, + queued, + deferred: orphaned.length - batch.length, + failed, + }; +} + +function flatten(run: RetentionRun): Record { + return { + ...Object.fromEntries( + Object.entries(run.idle).map(([key, value]) => [`idle.${key}`, value]), + ), + ...Object.fromEntries( + Object.entries(run.orphans).map(([key, value]) => [ + `orphans.${key}`, + value, + ]), + ), + }; +} diff --git a/packages/trpc/src/router/cloud-workspace/teardown-guard.ts b/packages/trpc/src/router/cloud-workspace/teardown-guard.ts new file mode 100644 index 00000000000..4b50e65b10b --- /dev/null +++ b/packages/trpc/src/router/cloud-workspace/teardown-guard.ts @@ -0,0 +1,38 @@ +import { db } from "@superset/db/client"; +import { cloudWorkspaces, environments } from "@superset/db/schema"; +import { and, inArray, isNull } from "drizzle-orm"; + +/** + * The names among `names` that a workspace row, in any status, or an + * unarchived environment still names. A real teardown clears the row's name + * in the write that commits to it, so a name still on a row is that row's: + * an archived row may be undone onto it, a failed one keeps it for diagnosis. + */ +export async function namedSandboxes( + names: readonly string[], +): Promise> { + if (names.length === 0) return new Set(); + const [workspaces, goldens] = await Promise.all([ + db + .select({ name: cloudWorkspaces.providerSandboxId }) + .from(cloudWorkspaces) + .where(inArray(cloudWorkspaces.providerSandboxId, [...names])), + db + .select({ name: environments.sourceRef }) + .from(environments) + .where( + and( + inArray(environments.sourceRef, [...names]), + isNull(environments.archivedAt), + ), + ), + ]); + return new Set([...workspaces, ...goldens].map((row) => row.name)); +} + +export async function liveReferences(args: { + providerSandboxId: string; +}): Promise<{ sandbox: boolean }> { + const named = await namedSandboxes([args.providerSandboxId]); + return { sandbox: named.has(args.providerSandboxId) }; +} diff --git a/packages/trpc/src/router/cloud-workspace/teardown.test.ts b/packages/trpc/src/router/cloud-workspace/teardown.test.ts new file mode 100644 index 00000000000..fa8711bd488 --- /dev/null +++ b/packages/trpc/src/router/cloud-workspace/teardown.test.ts @@ -0,0 +1,73 @@ +import { beforeEach, describe, expect, test } from "bun:test"; +import { stub } from "../../../test/stub"; +import * as sandbox from "../../lib/sandbox"; +import * as jobs from "./jobs"; +import { + TEARDOWN_ATTEMPTS, + TEARDOWN_JOB_PATH, + teardownSandbox, +} from "./teardown"; +import * as guard from "./teardown-guard"; + +let calls: string[] = []; +let namedOnRead: boolean[] = []; +let deleteFails = false; +let queued: Array<{ path: string; body: unknown; delaySeconds?: number }> = []; + +stub(guard, { + liveReferences: async () => ({ sandbox: namedOnRead.shift() ?? false }), +}); +stub(sandbox, { + stopSandbox: async (name: string) => { + calls.push(`stop:${name}`); + return { sessionId: null }; + }, + deleteSandbox: async (name: string) => { + if (deleteFails) throw new Error("vercel down"); + calls.push(`delete:${name}`); + }, +}); +stub(jobs, { + publishCloudWorkspaceJob: async (job: { + path: string; + body: unknown; + delaySeconds?: number; + }) => { + queued.push(job); + }, +}); + +beforeEach(() => { + calls = []; + namedOnRead = []; + deleteFails = false; + queued = []; +}); + +const input = { providerSandboxId: "ws-old", driveName: null, attempt: 0 }; + +describe("teardownSandbox", () => { + test("stops, then deletes a box no row names", async () => { + expect(await teardownSandbox(input)).toBe("deleted"); + expect(calls).toEqual(["stop:ws-old", "delete:ws-old"]); + }); + + test("keeps a box a row names by the time it would delete it", async () => { + namedOnRead = [false, true]; + expect(await teardownSandbox(input)).toBe("kept"); + expect(calls).toEqual(["stop:ws-old"]); + }); + + test("a failure requeues the next attempt, and the last attempt gives up", async () => { + deleteFails = true; + expect(await teardownSandbox(input)).toBe("retrying"); + expect(queued).toMatchObject([ + { path: TEARDOWN_JOB_PATH, body: { ...input, attempt: 1 } }, + ]); + queued = []; + expect( + await teardownSandbox({ ...input, attempt: TEARDOWN_ATTEMPTS - 1 }), + ).toBe("abandoned"); + expect(queued).toEqual([]); + }); +}); diff --git a/packages/trpc/src/router/cloud-workspace/teardown.ts b/packages/trpc/src/router/cloud-workspace/teardown.ts new file mode 100644 index 00000000000..25229e239f4 --- /dev/null +++ b/packages/trpc/src/router/cloud-workspace/teardown.ts @@ -0,0 +1,82 @@ +import * as Sentry from "@sentry/core"; +import { deleteSandbox, stopSandbox } from "../../lib/sandbox"; +import { publishCloudWorkspaceJob } from "./jobs"; +import { liveReferences } from "./teardown-guard"; + +export const TEARDOWN_JOB_PATH = "/api/cloud-workspaces/teardown"; + +export interface TeardownSandboxInput { + providerSandboxId: string; + /** Always null until workspace drives exist; #9847 tears down the drive here. */ + driveName: null; + attempt: number; +} + +export const TEARDOWN_RETRY_SECONDS = 30; +export const TEARDOWN_ATTEMPTS = 20; + +export type TeardownOutcome = "deleted" | "kept" | "retrying" | "abandoned"; + +/** + * Stops the box, then deletes it with its snapshots. Before each destructive + * call it reads the rows again: a box a workspace or a live environment names + * is kept. Any failure requeues the job. + */ +export async function teardownSandbox( + input: TeardownSandboxInput, +): Promise { + try { + return await teardownOnce(input); + } catch (error) { + Sentry.captureException(error, { extra: { ...input } }); + return retryOrAbandon(input, error); + } +} + +async function teardownOnce( + input: TeardownSandboxInput, +): Promise { + if ((await liveReferences(input)).sandbox) return kept(input); + await stopSandbox(input.providerSandboxId); + if ((await liveReferences(input)).sandbox) return kept(input); + await deleteSandbox(input.providerSandboxId); + console.log(`[cloud-workspace] teardown deleted ${input.providerSandboxId}`); + return "deleted"; +} + +function kept(input: TeardownSandboxInput): TeardownOutcome { + console.log( + `[cloud-workspace] teardown kept ${input.providerSandboxId}: a live row names it`, + ); + return "kept"; +} + +async function retryOrAbandon( + input: TeardownSandboxInput, + error: unknown, +): Promise { + if (input.attempt + 1 >= TEARDOWN_ATTEMPTS) { + Sentry.captureMessage( + `cloud workspace teardown gave up on ${input.providerSandboxId}`, + { level: "error", extra: { ...input, error: String(error) } }, + ); + return "abandoned"; + } + await queueTeardown( + { ...input, attempt: input.attempt + 1 }, + TEARDOWN_RETRY_SECONDS, + ); + return "retrying"; +} + +export function queueTeardown( + input: TeardownSandboxInput, + delaySeconds?: number, +): Promise { + return publishCloudWorkspaceJob({ + path: TEARDOWN_JOB_PATH, + body: input, + delaySeconds, + runLocally: teardownSandbox, + }); +} diff --git a/packages/trpc/src/router/cloud-workspace/transition.ts b/packages/trpc/src/router/cloud-workspace/transition.ts index 7c26d5cfbe4..c222c050a02 100644 --- a/packages/trpc/src/router/cloud-workspace/transition.ts +++ b/packages/trpc/src/router/cloud-workspace/transition.ts @@ -4,7 +4,7 @@ import type { CloudWorkspaceFailureReason } from "@superset/db/enums"; import type { CloudWorkspaceStatus } from "@superset/db/schema"; import { cloudWorkspaces } from "@superset/db/schema"; import { isCloudUsageRefusal } from "@superset/shared/billing"; -import { and, eq, inArray } from "drizzle-orm"; +import { and, eq, inArray, type SQL } from "drizzle-orm"; import { type ControlScope, controlTags, @@ -16,6 +16,7 @@ type Move = { id: string; from: readonly CloudWorkspaceStatus[]; set?: Partial; + where?: SQL; }; async function moveCloudWorkspace( @@ -28,6 +29,7 @@ async function moveCloudWorkspace( and( eq(cloudWorkspaces.id, args.id), inArray(cloudWorkspaces.status, [...args.from]), + args.where, ), ) .returning({ id: cloudWorkspaces.id }); @@ -53,31 +55,39 @@ export function transitionCloudWorkspace( ); } -/** An Autumn refusal is an expected state: it is logged as a warning and opens no Sentry issue. */ +/** + * An expected failure (an Autumn refusal, or `expected`) is logged as a + * warning and opens no Sentry issue. Returns whether the row moved. + */ export async function failCloudWorkspace(args: { scope: ControlScope; from: readonly CloudWorkspaceStatus[]; set?: Partial; + /** Further conditions the same update must meet. */ + where?: SQL; reason: CloudWorkspaceFailureReason; error: unknown; -}): Promise { + expected?: boolean; +}): Promise { const { scope } = args; const changed = await moveCloudWorkspace({ id: scope.cloudWorkspaceId, from: args.from, to: "failed", set: { ...args.set, failureReason: args.reason, failedAt: new Date() }, + where: args.where, }); nudge(scope.organizationId, "cloud_workspaces"); - if (!changed) return; - const refused = isCloudUsageRefusal(args.error); + if (!changed) return false; + const expected = args.expected || isCloudUsageRefusal(args.error); logControl( "workspace.failed", scope, { "failure.reason": args.reason }, - refused ? "warn" : "error", + expected ? "warn" : "error", ); - if (!refused) { + if (!expected) { Sentry.captureException(args.error, { tags: controlTags(scope) }); } + return true; } diff --git a/plans/done/20261118-1500-cloud-workspace-retention-reaper.md b/plans/done/20261118-1500-cloud-workspace-retention-reaper.md new file mode 100644 index 00000000000..eb7cf92c515 --- /dev/null +++ b/plans/done/20261118-1500-cloud-workspace-retention-reaper.md @@ -0,0 +1,83 @@ +# Make cloud workspace retention ours: a daily reaper for idle boxes and orphans + +This ExecPlan is a living document. The sections `Progress`, `Surprises & Discoveries`, `Decision Log`, and `Outcomes & Retrospective` must be kept up to date as work proceeds. + +Reference: This plan follows conventions from AGENTS.md and the ExecPlan template. + +## Purpose / Big Picture + +A cloud workspace is a row in `cloud_workspaces` backed by a Vercel sandbox ("box"). When the box stops, Vercel keeps a snapshot of its disk, and the next wake resumes from it. Today `provisionSandbox` in `packages/trpc/src/lib/sandbox/vercel.ts` creates workspace boxes with `snapshotExpiration: 30 days`, so Vercel deletes the snapshot of a box nobody used for 30 days. The app learns this only when a wake, an agent turn or an automation dispatch gets a 410 and calls `markSandboxUnavailable` (`router/cloud-workspace/wake.ts`), which moves the row to `failed` with reason `snapshot_expired`. A workspace nobody opens again never learns. The box itself, and boxes that no row names (orphans), stay in Vercel and use storage. + +After this change Vercel never expires a new workspace's snapshot. A daily job decides instead. It moves a workspace that has been asleep with no activity for 30 days to the same `failed`/`snapshot_expired` state the 410 path gives, takes the box name off the row in the same update, and queues a retried teardown that deletes the box. The same job queues teardown for orphan boxes. Each run logs and returns how many it retired, queued and skipped, and a Sentry cron monitor reports a run that did not happen. The customer sees the same 30-day retention as today. + +## Context and Orientation + +Apps and packages: `apps/api` (Next.js on Vercel, holds the QStash job routes) and `packages/trpc` (the cloud workspace logic). No tRPC procedure changes, no schema change, no migration. + +Terms. A "QStash job" is an HTTP POST that Upstash QStash delivers to `apps/api`, signed and retried. `publishCloudWorkspaceJob` (`packages/trpc/src/router/cloud-workspace/jobs.ts`) queues one, and runs it in-process when the API runs on localhost. A "scheduled job" in this repo is a route under `apps/api/src/app/api/cloud-workspaces/jobs//route.ts`. A QStash schedule in the Upstash console calls it, the route is wrapped in `Sentry.withMonitor` (a cron check-in), and `.github/workflows/publish-cloud-workspace-job.yml` can fire it once by hand. `reconcile-usage` is the model to copy. + +Names. A workspace box is named `sandboxNameFor(id)` or `nextSandboxNameFor(id)` (`router/cloud-workspace/provision.ts`) and tagged `{kind: "workspace", org, workspace, user}`. A golden (the box an environment forks from) is named `env-…`, is tagged `{kind: "environment", org}`, and is named by `environments.source_ref` of a fork environment. + +Last activity. `sandbox_expires_at` on an asleep row is when its last session ended. It is not reliable alone: in production on 2026-11-16 a one-off write moved it forward on 412 asleep rows past their last boot. Boot rows in `cloud_workspace_boots` (one per session: `started_at`, `stopped_at`) are written before Vercel resumes a box (`authorizeBoot` in `beforeResume`), so they are both the most accurate record and a guard against a wake in flight. The reaper therefore treats a row as idle since `cutoff` only when its `created_at` and `sandbox_expires_at` are older than `cutoff`, and it has no boot that is open or that started or stopped after `cutoff`. Taking the latest of these makes the reaper wait longer than Vercel when they disagree, never shorter. + +#9847 (draft, not merged) adds a `teardownSandbox` QStash job at `/api/cloud-workspaces/teardown` with payload `{providerSandboxId, driveName, attempt, keepDrive?}` and `retireAndTearDown(row, condition)`. Main has neither. This plan adds box-only versions with the same route, payload shape and function names, so #9847 replaces their bodies when it lands. That is the seam for drives. + +## Plan of Work + +1. `vercel.ts`: workspace boxes are created with `snapshotExpiration: 0` like goldens. Remove `WORKSPACE_SNAPSHOT_EXPIRATION_MS`. Add `listTaggedSandboxes(kind)`, which lists boxes of one tag kind with `createdAt`, `statusUpdatedAt`, `status` and `snapshotExpiration`. +2. `router/cloud-workspace/teardown.ts` (new): `teardownSandbox(input)` checks `liveReferences` (any workspace row in any status naming the box, or an unarchived environment's `source_ref`), stops the box, checks again, deletes it (`deleteSandbox`, which also deletes its snapshots). On error it requeues itself up to `TEARDOWN_ATTEMPTS` times. `queueTeardown(input, delaySeconds)`. `retireAndTearDown` is left to #9847; the reaper does its own atomic retire because it also changes the status. +3. `transition.ts`: `failCloudWorkspace` takes an optional `where` (extra conditions on the same update) and `expected` (log a warning, no Sentry exception), and returns whether it moved the row. Existing callers are unchanged. +4. `router/cloud-workspace/retention.ts` (new): `reapIdleCloudWorkspaces({now, production, deps})`. + - Idle stage: select up to `REAP_BATCH` asleep vercel rows idle since `now - 30d`. For each, a provider read skips a box that is running. Then one `failCloudWorkspace` update from `asleep`, with `where` = same name + still idle + not archived, sets `failure_reason = snapshot_expired`, `sandbox_url = null` and `provider_sandbox_id = nextSandboxNameFor(id)`. Only if it moved, queue the teardown of the old name. + - Orphan stage (only when `production`): list `workspace` and `environment` tagged boxes. Keep those in a settled status whose `createdAt` and `statusUpdatedAt` are older than `ORPHAN_MIN_AGE` (7 days). Drop every name a workspace row or an unarchived environment names, and queue teardown for up to `ORPHAN_BATCH`. + - Teardowns are staggered one per second (`delaySeconds = index`), far below Vercel's 20 deletions/s per team. + - It returns counts (`retired`, `orphansQueued`, `skipped{running,raced,orphanBatchFull,…}`) and logs them as one line. +5. `apps/api/.../cloud-workspaces/jobs/reap-retention/route.ts` (daily, `Sentry.withMonitor("reap-retention", …, "0 6 * * *")`) and `.../cloud-workspaces/teardown/route.ts`. Add `reap-retention` to the publish workflow's choices. Add package exports. +6. `packages/sandbox/src/backfill-snapshot-expiration.ts` (new, not run): a dry run by default. `--apply` calls `sandbox.update({snapshotExpiration: 0})` on workspace boxes whose `snapshotExpiration` is not 0, at 2/s. +7. Docs: an entry in `docs/cloud-sandbox-mismatches.md`. The TLA+ model is not edited; the PR says which action this maps to. + +## Validation and Acceptance + + cd packages/trpc && bun test src/router/cloud-workspace/retention.test.ts src/router/cloud-workspace/teardown.test.ts src/lib/sandbox/vercel.test.ts + bun run lint && bunx turbo run typecheck --filter=@superset/trpc --filter=@superset/api --filter=@superset/sandbox + +Each behaviour has one test, with an injected clock and no real waiting. + +## Idempotence and Recovery + +A run that stops halfway is safe to repeat. The retire update matches only a row that still names the old box. A teardown that finds the box gone does nothing. A box retired but never torn down (its job abandoned) is an orphan, so a later run queues it again. To roll back, pause the QStash schedule. Rows already retired stay `failed`, as after a 410. + +## Assumptions + +All deployments probably share one Vercel sandbox project, so an orphan seen from a non-production database may be a production box. The orphan stage therefore runs only when `VERCEL_ENV === "production"`. + +## Open Questions + +None blocking. The QStash schedule must be created in the Upstash console by someone with access, as for the other jobs. + +## Progress + +- [x] (2026-11-18 15:00Z) Discovery, production counts, plan. +- [x] (2026-11-18 22:10Z) Implementation: `snapshotExpiration: 0`, `teardownSandbox` (box only), `failCloudWorkspace` `where`/`expected`, `retention.ts`, both routes, the backfill script, docs. +- [x] (2026-11-18 22:20Z) trpc 1104 pass, sandbox 85 pass; lint; typecheck of trpc, api and sandbox. Ran the reaper's `idleSince` SQL read-only on production: 0 reapable at 30 days, 0 at 7, 518 at 2. +- [ ] Create the QStash schedule `0 6 * * *` for `/api/cloud-workspaces/jobs/reap-retention` in the Upstash console (needs console access; not done here). +- [ ] Run `bun run backfill-snapshot-expiration` dry, then `--apply` (not done here, on purpose). + +## Surprises & Discoveries + +- Observation: there is no `teardownSandbox` or box sweep on main; both are on #9847's branch only. +- Observation: `sandbox.update({snapshotExpiration})` exists in `@vercel/sandbox` 4.0.0 and does not resume a box. It sets the default for snapshots made from then on. The SDK has no call that changes an existing snapshot's expiry. +- Observation: production on 2026-11-18 had 857 asleep vercel rows, 0 idle for more than 30 days by either measure. By boot rows, 188 are idle for more than 7 days, and the oldest last boot is 2026-10-22. + +## Decision Log + +- Decision: reuse `failed` + `snapshot_expired`, no new enum value. + Rationale: the brief asks for the 410 path's terminal state, and a new value needs a migration and client copy. Date/Author: 2026-11-18, agent. +- Decision: last activity = latest of boot rows, `sandbox_expires_at` and `created_at`. + Rationale: deleting too late costs storage; deleting too early loses someone's work. Date/Author: 2026-11-18, agent. +- Decision: the orphan stage covers boxes only, not drives. + Rationale: on main no row names a workspace drive, so every `drive-` drive would look orphaned. Once #9847's columns exist, a drive stage must check them, and #9847's `teardownSandbox` deletes drives. Agent-tools drives are pruned by the release. Date/Author: 2026-11-18, agent. + +## Outcomes & Retrospective + +All seven parts of the brief are done except the drives. Workspace drives do not exist on main, so the drive half is a seam: `TeardownSandboxInput.driveName` is typed `null`, and #9847's version of `teardownSandbox` replaces this one at the same route with the same payload. The reaper's measure of last activity is deliberately later than Vercel's: the 412 rows whose `sandbox_expires_at` was moved forward on 2026-11-16 are reaped no earlier than 2026-12-16, and until then Vercel's own expiry and the 410 path still apply to them. From 832fd15948abacced422d2c964ecc413435cbf76 Mon Sep 17 00:00:00 2001 From: Satya Patel <14907857+saddlepaddle@users.noreply.github.com> Date: Wed, 18 Nov 2026 22:03:05 +0000 Subject: [PATCH 2/8] feat(cloud-workspace): run the retention reaper every 6 hours, 200 a batch Four runs of 200 clear 800 a day: production made a median 156 workspaces a day and at most 725. A run stops starting work after 200 s and reports the rest as deferred, so a backlog takes several runs inside the route's 300 s. --- .../jobs/reap-retention/route.ts | 4 +- docs/cloud-sandbox-mismatches.md | 2 +- .../router/cloud-workspace/retention.test.ts | 21 ++++++++++ .../src/router/cloud-workspace/retention.ts | 41 +++++++++++++++---- ...8-1500-cloud-workspace-retention-reaper.md | 9 ++-- 5 files changed, 63 insertions(+), 14 deletions(-) diff --git a/apps/api/src/app/api/cloud-workspaces/jobs/reap-retention/route.ts b/apps/api/src/app/api/cloud-workspaces/jobs/reap-retention/route.ts index f108794b8cc..84bc28ef098 100644 --- a/apps/api/src/app/api/cloud-workspaces/jobs/reap-retention/route.ts +++ b/apps/api/src/app/api/cloud-workspaces/jobs/reap-retention/route.ts @@ -9,7 +9,7 @@ export const dynamic = "force-dynamic"; export const maxDuration = 300; /** - * Run by a QStash schedule daily at 06:00 UTC. That schedule lives in the + * Run by a QStash schedule every 6 hours. That schedule lives in the * Upstash console, not in this repo. */ export async function POST(request: Request): Promise { @@ -23,7 +23,7 @@ export async function POST(request: Request): Promise { reapCloudWorkspaceRetention({ production: process.env.VERCEL_ENV === "production", }), - { schedule: { type: "crontab", value: "0 6 * * *" } }, + { schedule: { type: "crontab", value: "0 */6 * * *" } }, ); // Turbopack builds do not wrap route handlers, so nothing else flushes the closing check-in. await Sentry.flush(2_000).catch(() => false); diff --git a/docs/cloud-sandbox-mismatches.md b/docs/cloud-sandbox-mismatches.md index 29ac1a7783f..fdc3fc283ec 100644 --- a/docs/cloud-sandbox-mismatches.md +++ b/docs/cloud-sandbox-mismatches.md @@ -789,7 +789,7 @@ expired a workspace box's snapshot 30 days after its last use. Nothing told us: the row learned only from the 410 that its next wake, agent turn or automation dispatch got. A workspace nobody opened again stayed `asleep` over a box that could not resume, and the box itself stayed in Vercel. New workspace boxes are -now created with `snapshotExpiration: 0`. The daily `reap-retention` job +now created with `snapshotExpiration: 0`. The `reap-retention` job, every 6 hours, (`router/cloud-workspace/retention.ts`) does three things. It moves a workspace that is asleep and unused for 30 days to `failed`/`snapshot_expired`, the same state as the 410. It gives the row a new box name in that same update. Then it diff --git a/packages/trpc/src/router/cloud-workspace/retention.test.ts b/packages/trpc/src/router/cloud-workspace/retention.test.ts index 00170b09d4f..533e576ed57 100644 --- a/packages/trpc/src/router/cloud-workspace/retention.test.ts +++ b/packages/trpc/src/router/cloud-workspace/retention.test.ts @@ -5,6 +5,7 @@ import { ORPHAN_BATCH, RETENTION_IDLE_MS, type RetentionDeps, + RUN_BUDGET_MS, reapCloudWorkspaceRetention, retireIdleWorkspace, } from "./retention"; @@ -167,6 +168,26 @@ describe("reapCloudWorkspaceRetention", () => { expect(queued).toHaveLength(ORPHAN_BATCH); expect(run.orphans).toMatchObject({ queued: ORPHAN_BATCH, deferred: 3 }); }); + + test("stops starting work when its time is up and leaves the rest for the next run", async () => { + let time = 0; + const { deps, retired } = fakeDeps({ + findIdleWorkspaces: async () => [idleRow(1), idleRow(2), idleRow(3)], + retireIdleWorkspace: async (row) => { + retired.push(row.id); + time += RUN_BUDGET_MS + 1; + return true; + }, + }); + const run = await reapCloudWorkspaceRetention({ + now: NOW, + production: false, + deps, + clock: () => time, + }); + expect(retired).toEqual(["ws_1"]); + expect(run.idle).toMatchObject({ retired: 1, deferred: 2 }); + }); }); describe("retireIdleWorkspace", () => { diff --git a/packages/trpc/src/router/cloud-workspace/retention.ts b/packages/trpc/src/router/cloud-workspace/retention.ts index 52711576091..2bb984653b6 100644 --- a/packages/trpc/src/router/cloud-workspace/retention.ts +++ b/packages/trpc/src/router/cloud-workspace/retention.ts @@ -20,8 +20,14 @@ const DAY_MS = 24 * 60 * 60 * 1000; export const RETENTION_IDLE_MS = 30 * DAY_MS; /** Longer than a release or a promote takes to write the name of a golden it built. */ export const ORPHAN_MIN_AGE_MS = 7 * DAY_MS; -export const REAP_BATCH = 50; -export const ORPHAN_BATCH = 50; +/** + * Every 6 hours, so 800 a day: production made a median 156 workspaces a day + * and at most 725 in the 30 days to 2026-11-18. A backlog takes more runs. + */ +export const REAP_BATCH = 200; +export const ORPHAN_BATCH = 200; +/** Stops starting new work here, well inside the route's 300 s; the rest waits for the next run. */ +export const RUN_BUDGET_MS = 200_000; /** One delete a second: far under Vercel's 20 deletions a second per team. */ const TEARDOWN_SPACING_SECONDS = 1; const NAME_LOOKUP_CHUNK = 500; @@ -156,6 +162,7 @@ export interface RetentionRun { retired: number; skippedRunning: number; skippedChanged: number; + deferred: number; failed: number; }; orphans: @@ -170,7 +177,7 @@ export interface RetentionRun { } /** - * The one place a workspace box is deleted for age. Daily: a workspace asleep + * The one place a workspace box is deleted for age. Every 6 hours: a workspace asleep * and unused for `RETENTION_IDLE_MS` fails as `snapshot_expired`, exactly as a * wake that met Vercel's expiry did, and its box is torn down. Then, in * production only, boxes we tagged that no row names are torn down. Every @@ -181,18 +188,22 @@ export async function reapCloudWorkspaceRetention({ now = Date.now(), production, deps = defaultDeps, + clock = Date.now, }: { now?: number; production: boolean; deps?: RetentionDeps; + clock?: () => number; }): Promise { + const startedAt = clock(); + const outOfTime = () => clock() - startedAt > RUN_BUDGET_MS; let slot = 0; const queue = (name: string) => deps.queueTeardown(name, slot++ * TEARDOWN_SPACING_SECONDS); const retiredNames = new Set(); - const idle = await reapIdle({ now, deps, queue, retiredNames }); + const idle = await reapIdle({ now, deps, queue, retiredNames, outOfTime }); const orphans = production - ? await reapOrphans({ now, deps, queue, retiredNames }) + ? await reapOrphans({ now, deps, queue, retiredNames, outOfTime }) : ({ skipped: "not production" } as const); const run = { idle, orphans }; console.log("[cloud-workspace] retention run", JSON.stringify(run)); @@ -211,11 +222,13 @@ async function reapIdle({ deps, queue, retiredNames, + outOfTime, }: { now: number; deps: RetentionDeps; queue: (name: string) => Promise; retiredNames: Set; + outOfTime: () => boolean; }): Promise { const cutoff = new Date(now - RETENTION_IDLE_MS); const rows = await deps.findIdleWorkspaces(cutoff, REAP_BATCH); @@ -224,9 +237,14 @@ async function reapIdle({ retired: 0, skippedRunning: 0, skippedChanged: 0, + deferred: 0, failed: 0, }; - for (const row of rows) { + for (const [index, row] of rows.entries()) { + if (outOfTime()) { + counts.deferred = rows.length - index; + break; + } try { if (await deps.boxIsRunning(row.providerSandboxId)) { counts.skippedRunning++; @@ -263,11 +281,13 @@ async function reapOrphans({ deps, queue, retiredNames, + outOfTime, }: { now: number; deps: RetentionDeps; queue: (name: string) => Promise; retiredNames: Set; + outOfTime: () => boolean; }): Promise> { const boxes = [ ...(await deps.listTaggedSandboxes("workspace")), @@ -281,7 +301,12 @@ async function reapOrphans({ const batch = orphaned.slice(0, ORPHAN_BATCH); let queued = 0; let failed = 0; - for (const name of batch) { + let unstarted = 0; + for (const [index, name] of batch.entries()) { + if (outOfTime()) { + unstarted = batch.length - index; + break; + } try { await queue(name); queued++; @@ -294,7 +319,7 @@ async function reapOrphans({ listed: boxes.length, orphaned: orphaned.length, queued, - deferred: orphaned.length - batch.length, + deferred: orphaned.length - batch.length + unstarted, failed, }; } diff --git a/plans/done/20261118-1500-cloud-workspace-retention-reaper.md b/plans/done/20261118-1500-cloud-workspace-retention-reaper.md index eb7cf92c515..44753490493 100644 --- a/plans/done/20261118-1500-cloud-workspace-retention-reaper.md +++ b/plans/done/20261118-1500-cloud-workspace-retention-reaper.md @@ -8,7 +8,7 @@ Reference: This plan follows conventions from AGENTS.md and the ExecPlan templat A cloud workspace is a row in `cloud_workspaces` backed by a Vercel sandbox ("box"). When the box stops, Vercel keeps a snapshot of its disk, and the next wake resumes from it. Today `provisionSandbox` in `packages/trpc/src/lib/sandbox/vercel.ts` creates workspace boxes with `snapshotExpiration: 30 days`, so Vercel deletes the snapshot of a box nobody used for 30 days. The app learns this only when a wake, an agent turn or an automation dispatch gets a 410 and calls `markSandboxUnavailable` (`router/cloud-workspace/wake.ts`), which moves the row to `failed` with reason `snapshot_expired`. A workspace nobody opens again never learns. The box itself, and boxes that no row names (orphans), stay in Vercel and use storage. -After this change Vercel never expires a new workspace's snapshot. A daily job decides instead. It moves a workspace that has been asleep with no activity for 30 days to the same `failed`/`snapshot_expired` state the 410 path gives, takes the box name off the row in the same update, and queues a retried teardown that deletes the box. The same job queues teardown for orphan boxes. Each run logs and returns how many it retired, queued and skipped, and a Sentry cron monitor reports a run that did not happen. The customer sees the same 30-day retention as today. +After this change Vercel never expires a new workspace's snapshot. A job that runs every 6 hours decides instead. It moves a workspace that has been asleep with no activity for 30 days to the same `failed`/`snapshot_expired` state the 410 path gives, takes the box name off the row in the same update, and queues a retried teardown that deletes the box. The same job queues teardown for orphan boxes. Each run logs and returns how many it retired, queued and skipped, and a Sentry cron monitor reports a run that did not happen. The customer sees the same 30-day retention as today. ## Context and Orientation @@ -32,7 +32,7 @@ Last activity. `sandbox_expires_at` on an asleep row is when its last session en - Orphan stage (only when `production`): list `workspace` and `environment` tagged boxes. Keep those in a settled status whose `createdAt` and `statusUpdatedAt` are older than `ORPHAN_MIN_AGE` (7 days). Drop every name a workspace row or an unarchived environment names, and queue teardown for up to `ORPHAN_BATCH`. - Teardowns are staggered one per second (`delaySeconds = index`), far below Vercel's 20 deletions/s per team. - It returns counts (`retired`, `orphansQueued`, `skipped{running,raced,orphanBatchFull,…}`) and logs them as one line. -5. `apps/api/.../cloud-workspaces/jobs/reap-retention/route.ts` (daily, `Sentry.withMonitor("reap-retention", …, "0 6 * * *")`) and `.../cloud-workspaces/teardown/route.ts`. Add `reap-retention` to the publish workflow's choices. Add package exports. +5. `apps/api/.../cloud-workspaces/jobs/reap-retention/route.ts` (every 6 hours, `Sentry.withMonitor("reap-retention", …, "0 */6 * * *")`) and `.../cloud-workspaces/teardown/route.ts`. Add `reap-retention` to the publish workflow's choices. Add package exports. 6. `packages/sandbox/src/backfill-snapshot-expiration.ts` (new, not run): a dry run by default. `--apply` calls `sandbox.update({snapshotExpiration: 0})` on workspace boxes whose `snapshotExpiration` is not 0, at 2/s. 7. Docs: an entry in `docs/cloud-sandbox-mismatches.md`. The TLA+ model is not edited; the PR says which action this maps to. @@ -60,7 +60,7 @@ None blocking. The QStash schedule must be created in the Upstash console by som - [x] (2026-11-18 15:00Z) Discovery, production counts, plan. - [x] (2026-11-18 22:10Z) Implementation: `snapshotExpiration: 0`, `teardownSandbox` (box only), `failCloudWorkspace` `where`/`expected`, `retention.ts`, both routes, the backfill script, docs. - [x] (2026-11-18 22:20Z) trpc 1104 pass, sandbox 85 pass; lint; typecheck of trpc, api and sandbox. Ran the reaper's `idleSince` SQL read-only on production: 0 reapable at 30 days, 0 at 7, 518 at 2. -- [ ] Create the QStash schedule `0 6 * * *` for `/api/cloud-workspaces/jobs/reap-retention` in the Upstash console (needs console access; not done here). +- [ ] Create the QStash schedule `0 */6 * * *` for `/api/cloud-workspaces/jobs/reap-retention` in the Upstash console (needs console access; not done here). - [ ] Run `bun run backfill-snapshot-expiration` dry, then `--apply` (not done here, on purpose). ## Surprises & Discoveries @@ -78,6 +78,9 @@ None blocking. The QStash schedule must be created in the Upstash console by som - Decision: the orphan stage covers boxes only, not drives. Rationale: on main no row names a workspace drive, so every `drive-` drive would look orphaned. Once #9847's columns exist, a drive stage must check them, and #9847's `teardownSandbox` deletes drives. Agent-tools drives are pruned by the release. Date/Author: 2026-11-18, agent. +- Decision: run every 6 hours (`0 */6 * * *`) with batches of 200 idle rows and 200 orphans, and stop starting work after 200 s of a run (`RUN_BUDGET_MS`). + Rationale: Satya asked for 6-hourly runs sized to clear a normal day with room to spare. Production made a median 156 workspaces a day and at most 725 in the 30 days to 2026-11-18. Four runs of 200 is 800 a day, so even the peak day clears within one day, and a backlog takes several runs. The time budget keeps a large run inside the route's 300 s; the rest is reported as `deferred`. Date/Author: 2026-11-18, Satya via orchestrator. + ## Outcomes & Retrospective All seven parts of the brief are done except the drives. Workspace drives do not exist on main, so the drive half is a seam: `TeardownSandboxInput.driveName` is typed `null`, and #9847's version of `teardownSandbox` replaces this one at the same route with the same payload. The reaper's measure of last activity is deliberately later than Vercel's: the 412 rows whose `sandbox_expires_at` was moved forward on 2026-11-16 are reaped no earlier than 2026-12-16, and until then Vercel's own expiry and the 410 path still apply to them. From 346682f0b3dfe88a2d30ad62a4ae5e5dc46028dc Mon Sep 17 00:00:00 2001 From: Satya Patel <14907857+saddlepaddle@users.noreply.github.com> Date: Wed, 18 Nov 2026 22:10:28 +0000 Subject: [PATCH 3/8] feat(cloud-workspace): reap archived boxes whose 7-day delete was lost With snapshotExpiration 0, Vercel no longer cleans up after a lost archive delete. In production each run lists every box once. An archived row past grace + 1 day whose box is still listed is deleted through the archive's own delete stage, or through a new guard (archived, no deleted_at, updated_at older) for rows archived before deleted_at existed. At most 20 a run, counted as found / alreadyDeleted / deleted / skipped / deferred / failed. --- docs/cloud-sandbox-mismatches.md | 9 +- packages/trpc/src/lib/sandbox/index.ts | 4 +- packages/trpc/src/lib/sandbox/vercel.ts | 20 +- .../src/router/cloud-workspace/reap.test.ts | 45 +++- .../trpc/src/router/cloud-workspace/reap.ts | 52 +++-- .../router/cloud-workspace/retention.test.ts | 119 ++++++++-- .../src/router/cloud-workspace/retention.ts | 208 ++++++++++++++++-- ...8-1500-cloud-workspace-retention-reaper.md | 3 + 8 files changed, 392 insertions(+), 68 deletions(-) diff --git a/docs/cloud-sandbox-mismatches.md b/docs/cloud-sandbox-mismatches.md index fdc3fc283ec..ab68f3cf9f7 100644 --- a/docs/cloud-sandbox-mismatches.md +++ b/docs/cloud-sandbox-mismatches.md @@ -796,7 +796,14 @@ state as the 410. It gives the row a new box name in that same update. Then it queues `teardownSandbox`, which deletes the box only if no row names it. "Unused" means no boot started or stopped since then, with `sandbox_expires_at` and `created_at` older too, so a wake in flight (its boot row is written before -Vercel resumes) keeps the box. In production only, the same job tears down +Vercel resumes) keeps the box. Archive deletes its box at the end of the +7-day grace through one delayed QStash message, and Vercel's 30-day expiry used +to clean up after a lost one. That backstop is gone, so in production the job +also deletes an archived box still listed a day after its grace, through the +archive's own delete stage (`reapUndatedArchivedCloudWorkspace` for the rows +archived before `deleted_at` existed). It matches rows against one listing of +every box, so a box already gone costs no provider call. In production only, +the same job also tears down tagged boxes that no row and no unarchived environment names, once they are 7 days old: every deployment shares one Vercel project, so a dev or preview database would see production's boxes as orphans. Boxes created before this diff --git a/packages/trpc/src/lib/sandbox/index.ts b/packages/trpc/src/lib/sandbox/index.ts index 2f6de7e6908..b3a7985f15a 100644 --- a/packages/trpc/src/lib/sandbox/index.ts +++ b/packages/trpc/src/lib/sandbox/index.ts @@ -59,9 +59,10 @@ export { describeSandbox, findSandboxSession, HOST_SERVICE_PORT, + type ListedSandbox, listActiveWorkspaceSandboxes, + listAllSandboxes, listSandboxSessions, - listTaggedSandboxes, MEMORY_MB_PER_VCPU, promoteSandboxToEnvironment, provisionSandbox, @@ -76,7 +77,6 @@ export { stopAndSnapshot, stopSandbox, stripWorkspaceIdentity, - type TaggedSandbox, topUpSandboxSession, waitForStopSnapshot, wakeSandbox, diff --git a/packages/trpc/src/lib/sandbox/vercel.ts b/packages/trpc/src/lib/sandbox/vercel.ts index fbed4185c2f..ba74cbbb802 100644 --- a/packages/trpc/src/lib/sandbox/vercel.ts +++ b/packages/trpc/src/lib/sandbox/vercel.ts @@ -987,29 +987,25 @@ export async function listActiveWorkspaceSandboxes( return found; } -export interface TaggedSandbox { +export interface ListedSandbox { name: string; + /** Our `kind` tag; absent on a box made before tags or not made by us. */ + kind: string | undefined; createdAt: number; statusUpdatedAt: number; status: string; - snapshotExpiration: number | undefined; } -/** Every box we tagged with this kind, whatever its age. */ -export async function listTaggedSandboxes( - kind: "workspace" | "environment", -): Promise { - const found: TaggedSandbox[] = []; - for await (const sandbox of await Sandbox.list({ - ...credentials(), - tags: { kind }, - })) { +/** Every box in the project, whatever its age or tags. */ +export async function listAllSandboxes(): Promise { + const found: ListedSandbox[] = []; + for await (const sandbox of await Sandbox.list(credentials())) { found.push({ name: sandbox.name, + kind: sandbox.tags?.kind, createdAt: sandbox.createdAt, statusUpdatedAt: sandbox.statusUpdatedAt ?? sandbox.updatedAt, status: sandbox.status, - snapshotExpiration: sandbox.snapshotExpiration, }); } return found; diff --git a/packages/trpc/src/router/cloud-workspace/reap.test.ts b/packages/trpc/src/router/cloud-workspace/reap.test.ts index b200c936b57..c1723e69bf0 100644 --- a/packages/trpc/src/router/cloud-workspace/reap.test.ts +++ b/packages/trpc/src/router/cloud-workspace/reap.test.ts @@ -13,6 +13,7 @@ type Row = { deletedAt: Date | null; provider: string; providerSandboxId: string; + updatedAt?: Date; }; type Stage = "stop" | "delete"; @@ -100,7 +101,11 @@ stub(meter, { }, }); -const { queueReap, reapArchivedCloudWorkspace } = await import("./reap"); +const { + queueReap, + reapArchivedCloudWorkspace, + reapUndatedArchivedCloudWorkspace, +} = await import("./reap"); const archivedAt = new Date("2026-10-01T00:00:00.000Z"); const input = { @@ -313,3 +318,41 @@ describe("reapArchivedCloudWorkspace", () => { expect(sandboxCalls).toEqual([]); }); }); + +describe("reapUndatedArchivedCloudWorkspace", () => { + test("deletes the box of an archived row with no archive date only while it is still archived and unchanged", async () => { + const archivedBefore = new Date("2026-11-10T00:00:00.000Z"); + const undated = { + id: input.cloudWorkspaceId, + status: "archived", + deletedAt: null, + provider: "vercel", + providerSandboxId: "ws-undated", + updatedAt: new Date("2026-10-01T00:00:00.000Z"), + }; + sandboxCalls = []; + row = { ...undated, status: "asleep" }; + expect( + await reapUndatedArchivedCloudWorkspace({ + cloudWorkspaceId: undated.id, + archivedBefore, + }), + ).toBe("skipped"); + row = { ...undated, updatedAt: new Date("2026-11-12T00:00:00.000Z") }; + expect( + await reapUndatedArchivedCloudWorkspace({ + cloudWorkspaceId: undated.id, + archivedBefore, + }), + ).toBe("skipped"); + expect(sandboxCalls).toEqual([]); + row = undated; + expect( + await reapUndatedArchivedCloudWorkspace({ + cloudWorkspaceId: undated.id, + archivedBefore, + }), + ).toBe("reaped"); + expect(sandboxCalls).toEqual(["delete:ws-undated"]); + }); +}); diff --git a/packages/trpc/src/router/cloud-workspace/reap.ts b/packages/trpc/src/router/cloud-workspace/reap.ts index 1f815c4a577..a5db0fe33d2 100644 --- a/packages/trpc/src/router/cloud-workspace/reap.ts +++ b/packages/trpc/src/router/cloud-workspace/reap.ts @@ -89,19 +89,7 @@ export async function reapArchivedCloudWorkspace( return "skipped"; } if (input.stage === "delete") { - await recordBootEnd( - { cloudWorkspaceId: row.id, organizationId: row.organizationId }, - { sessionId: null, reason: "archived" }, - ); - await settleWorkspaceNow(row.id).catch((error) => - console.error(`[cloud-workspace] ${row.id} final settle failed`, error), - ); - await deleteSandbox(row.providerSandboxId); - await reapWorkspaceFiles(row.id); - logControl("archive.deleted", { - cloudWorkspaceId: row.id, - organizationId: row.organizationId, - }); + await deleteArchivedBox(row); return "reaped"; } if (!(await runArchiveHook(row, input))) return "skipped"; @@ -121,6 +109,44 @@ export async function reapArchivedCloudWorkspace( return "stopped"; } +async function deleteArchivedBox(row: CloudWorkspaceRow): Promise { + await recordBootEnd( + { cloudWorkspaceId: row.id, organizationId: row.organizationId }, + { sessionId: null, reason: "archived" }, + ); + await settleWorkspaceNow(row.id).catch((error) => + console.error(`[cloud-workspace] ${row.id} final settle failed`, error), + ); + await deleteSandbox(row.providerSandboxId); + await reapWorkspaceFiles(row.id); + logControl("archive.deleted", { + cloudWorkspaceId: row.id, + organizationId: row.organizationId, + }); +} + +/** + * The delete stage for an archived row with no archive date: no reap was + * ever queued for it, so it has no `archivedAt` to match. Deletes only while + * the row is still archived, undated, and unchanged since `archivedBefore`. + */ +export async function reapUndatedArchivedCloudWorkspace(input: { + cloudWorkspaceId: string; + archivedBefore: Date; +}): Promise<"reaped" | "skipped"> { + const row = await findRow(input.cloudWorkspaceId); + if ( + row?.status !== "archived" || + row.deletedAt !== null || + row.provider !== "vercel" || + row.updatedAt >= input.archivedBefore + ) { + return "skipped"; + } + await deleteArchivedBox(row); + return "reaped"; +} + /** Drops the files sent to a workspace whose box is gone for good. */ export async function reapWorkspaceFiles(cloudWorkspaceId: string) { await reapOrphanFiles( diff --git a/packages/trpc/src/router/cloud-workspace/retention.test.ts b/packages/trpc/src/router/cloud-workspace/retention.test.ts index 533e576ed57..5f5aaa15fe4 100644 --- a/packages/trpc/src/router/cloud-workspace/retention.test.ts +++ b/packages/trpc/src/router/cloud-workspace/retention.test.ts @@ -1,7 +1,8 @@ import { describe, expect, test } from "bun:test"; import { stub } from "../../../test/stub"; -import type { TaggedSandbox } from "../../lib/sandbox"; +import type { ListedSandbox } from "../../lib/sandbox"; import { + ARCHIVED_BACKSTOP_AFTER_MS, ORPHAN_BATCH, RETENTION_IDLE_MS, type RetentionDeps, @@ -20,18 +21,22 @@ const idleRow = (n: number) => ({ providerSandboxId: `ws-box${n}`, }); -const box = (name: string, overrides: Partial = {}) => ({ +const box = ( + name: string, + overrides: Partial = {}, +): ListedSandbox => ({ name, + kind: "workspace", createdAt: NOW - 60 * DAY, statusUpdatedAt: NOW - 40 * DAY, status: "stopped", - snapshotExpiration: 0, ...overrides, }); function fakeDeps(overrides: Partial = {}) { const queued: Array<[string, number]> = []; const retired: string[] = []; + const reaped: unknown[] = []; const deps: RetentionDeps = { findIdleWorkspaces: async () => [], retireIdleWorkspace: async (row) => { @@ -39,14 +44,24 @@ function fakeDeps(overrides: Partial = {}) { return true; }, boxIsRunning: async () => false, - listTaggedSandboxes: async () => [], + listAllSandboxes: async () => [], namedSandboxes: async () => new Set(), + countArchivedPastGrace: async () => 0, + findArchivedPastGraceAmong: async () => [], + reapArchived: async (input) => { + reaped.push(input); + return "reaped"; + }, + reapUndatedArchived: async (input) => { + reaped.push(input); + return "reaped"; + }, queueTeardown: async (name, delaySeconds) => { queued.push([name, delaySeconds]); }, ...overrides, }; - return { deps, queued, retired }; + return { deps, queued, retired, reaped }; } describe("reapCloudWorkspaceRetention", () => { @@ -104,17 +119,16 @@ describe("reapCloudWorkspaceRetention", () => { test("queues only old, settled boxes no row names, and not one it just retired", async () => { const { deps, queued } = fakeDeps({ findIdleWorkspaces: async () => [idleRow(1)], - listTaggedSandboxes: async (kind) => - kind === "workspace" - ? [ - box("ws-box1"), - box("ws-orphan"), - box("ws-named"), - box("ws-young", { createdAt: NOW - DAY }), - box("ws-recent", { statusUpdatedAt: NOW - DAY }), - box("ws-running", { status: "running" }), - ] - : [box("env-orphan")], + listAllSandboxes: async () => [ + box("ws-box1"), + box("ws-orphan"), + box("ws-named"), + box("ws-young", { createdAt: NOW - DAY }), + box("ws-recent", { statusUpdatedAt: NOW - DAY }), + box("ws-running", { status: "running" }), + box("env-orphan", { kind: "environment" }), + box("someone-elses", { kind: undefined }), + ], namedSandboxes: async () => new Set(["ws-named"]), }); const run = await reapCloudWorkspaceRetention({ @@ -136,13 +150,16 @@ describe("reapCloudWorkspaceRetention", () => { }); }); - test("outside production it never looks for orphans", async () => { + test("outside production it touches no archived box and looks for no orphan", async () => { let listed = false; - const { deps } = fakeDeps({ - listTaggedSandboxes: async () => { + const { deps, reaped } = fakeDeps({ + listAllSandboxes: async () => { listed = true; return [box("ws-orphan")]; }, + findArchivedPastGraceAmong: async () => [ + { id: "ws_a", providerSandboxId: "ws-a", deletedAt: new Date(0) }, + ], }); const run = await reapCloudWorkspaceRetention({ now: NOW, @@ -150,15 +167,77 @@ describe("reapCloudWorkspaceRetention", () => { deps, }); expect(listed).toBe(false); + expect(reaped).toEqual([]); + expect(run.archived).toEqual({ skipped: "not production" }); expect(run.orphans).toEqual({ skipped: "not production" }); }); + test("deletes an archived box still there past its grace through the archive's own delete, by the guard that fits the row", async () => { + let cutoff: Date | undefined; + let askedAbout: string[] = []; + const archivedAt = new Date("2026-10-01T00:00:00.000Z"); + const { deps, reaped } = fakeDeps({ + listAllSandboxes: async () => [box("ws-dated"), box("ws-undated")], + countArchivedPastGrace: async (at) => { + cutoff = at; + return 40; + }, + findArchivedPastGraceAmong: async (_at, names) => { + askedAbout = names; + return [ + { id: "ws_d", providerSandboxId: "ws-dated", deletedAt: archivedAt }, + { id: "ws_u", providerSandboxId: "ws-undated", deletedAt: null }, + ]; + }, + }); + const run = await reapCloudWorkspaceRetention({ + now: NOW, + production: true, + deps, + }); + expect(cutoff?.getTime()).toBe(NOW - ARCHIVED_BACKSTOP_AFTER_MS); + expect(askedAbout).toEqual(["ws-dated", "ws-undated"]); + expect(reaped).toEqual([ + { + cloudWorkspaceId: "ws_d", + archivedAt: archivedAt.toISOString(), + stage: "delete", + }, + { cloudWorkspaceId: "ws_u", archivedBefore: cutoff }, + ]); + expect(run.archived).toEqual({ + found: 40, + alreadyDeleted: 38, + deleted: 2, + skipped: 0, + deferred: 0, + failed: 0, + }); + }); + + test("an archived row unarchived or changed before its delete is counted as skipped", async () => { + const { deps } = fakeDeps({ + listAllSandboxes: async () => [box("ws-dated")], + countArchivedPastGrace: async () => 1, + findArchivedPastGraceAmong: async () => [ + { id: "ws_d", providerSandboxId: "ws-dated", deletedAt: new Date(0) }, + ], + reapArchived: async () => "skipped", + }); + const run = await reapCloudWorkspaceRetention({ + now: NOW, + production: true, + deps, + }); + expect(run.archived).toMatchObject({ deleted: 0, skipped: 1 }); + }); + test("queues at most a batch of orphans a run and counts the rest", async () => { const many = Array.from({ length: ORPHAN_BATCH + 3 }, (_, i) => box(`ws-orphan${i}`), ); const { deps, queued } = fakeDeps({ - listTaggedSandboxes: async (kind) => (kind === "workspace" ? many : []), + listAllSandboxes: async () => many, }); const run = await reapCloudWorkspaceRetention({ now: NOW, diff --git a/packages/trpc/src/router/cloud-workspace/retention.ts b/packages/trpc/src/router/cloud-workspace/retention.ts index 2bb984653b6..f1dc71b6b4a 100644 --- a/packages/trpc/src/router/cloud-workspace/retention.ts +++ b/packages/trpc/src/router/cloud-workspace/retention.ts @@ -1,14 +1,31 @@ import * as Sentry from "@sentry/core"; import { db } from "@superset/db/client"; import { cloudWorkspaceBoots, cloudWorkspaces } from "@superset/db/schema"; -import { and, asc, eq, isNull, lt, or, type SQL, sql } from "drizzle-orm"; +import { + and, + asc, + count, + eq, + inArray, + isNotNull, + isNull, + lt, + or, + type SQL, + sql, +} from "drizzle-orm"; import { describeSandbox, - listTaggedSandboxes, + type ListedSandbox, + listAllSandboxes, SandboxUnavailableError, - type TaggedSandbox, } from "../../lib/sandbox"; import { nextSandboxNameFor } from "./provision"; +import { + ARCHIVE_GRACE_SECONDS, + reapArchivedCloudWorkspace, + reapUndatedArchivedCloudWorkspace, +} from "./reap"; import { queueTeardown } from "./teardown"; import { namedSandboxes } from "./teardown-guard"; import { failCloudWorkspace } from "./transition"; @@ -26,6 +43,10 @@ export const ORPHAN_MIN_AGE_MS = 7 * DAY_MS; */ export const REAP_BATCH = 200; export const ORPHAN_BATCH = 200; +/** A backstop for a lost archive delete, which deletes inline: few a run. */ +export const ARCHIVED_BATCH = 20; +/** The archive's own delete runs at the grace; a day later it has had its retries. */ +export const ARCHIVED_BACKSTOP_AFTER_MS = ARCHIVE_GRACE_SECONDS * 1000 + DAY_MS; /** Stops starting new work here, well inside the route's 300 s; the rest waits for the next run. */ export const RUN_BUDGET_MS = 200_000; /** One delete a second: far under Vercel's 20 deletions a second per team. */ @@ -119,6 +140,66 @@ async function boxIsRunning(providerSandboxId: string): Promise { } } +function archivedPastGrace(cutoff: Date): SQL { + return and( + eq(cloudWorkspaces.provider, "vercel"), + eq(cloudWorkspaces.status, "archived"), + or( + and( + isNotNull(cloudWorkspaces.deletedAt), + lt(cloudWorkspaces.deletedAt, cutoff), + ), + and( + isNull(cloudWorkspaces.deletedAt), + lt(cloudWorkspaces.updatedAt, cutoff), + ), + ), + ) as SQL; +} + +async function countArchivedPastGrace(cutoff: Date): Promise { + const [row] = await db + .select({ n: count() }) + .from(cloudWorkspaces) + .where(archivedPastGrace(cutoff)); + return row?.n ?? 0; +} + +export interface ArchivedWorkspace { + id: string; + providerSandboxId: string; + deletedAt: Date | null; +} + +/** The archived rows past their grace whose box is among `names`. */ +async function findArchivedPastGraceAmong( + cutoff: Date, + names: string[], +): Promise { + const found: ArchivedWorkspace[] = []; + for (let i = 0; i < names.length; i += NAME_LOOKUP_CHUNK) { + found.push( + ...(await db + .select({ + id: cloudWorkspaces.id, + providerSandboxId: cloudWorkspaces.providerSandboxId, + deletedAt: cloudWorkspaces.deletedAt, + }) + .from(cloudWorkspaces) + .where( + and( + archivedPastGrace(cutoff), + inArray( + cloudWorkspaces.providerSandboxId, + names.slice(i, i + NAME_LOOKUP_CHUNK), + ), + ), + )), + ); + } + return found; +} + async function namedAmong(names: string[]): Promise> { const named = new Set(); for (let i = 0; i < names.length; i += NAME_LOOKUP_CHUNK) { @@ -135,8 +216,12 @@ export interface RetentionDeps { findIdleWorkspaces: typeof findIdleWorkspaces; retireIdleWorkspace: typeof retireIdleWorkspace; boxIsRunning: typeof boxIsRunning; - listTaggedSandboxes: typeof listTaggedSandboxes; + listAllSandboxes: typeof listAllSandboxes; namedSandboxes: typeof namedAmong; + countArchivedPastGrace: typeof countArchivedPastGrace; + findArchivedPastGraceAmong: typeof findArchivedPastGraceAmong; + reapArchived: typeof reapArchivedCloudWorkspace; + reapUndatedArchived: typeof reapUndatedArchivedCloudWorkspace; queueTeardown: ( providerSandboxId: string, delaySeconds: number, @@ -147,8 +232,12 @@ const defaultDeps: RetentionDeps = { findIdleWorkspaces, retireIdleWorkspace, boxIsRunning, - listTaggedSandboxes, + listAllSandboxes, namedSandboxes: namedAmong, + countArchivedPastGrace, + findArchivedPastGraceAmong, + reapArchived: reapArchivedCloudWorkspace, + reapUndatedArchived: reapUndatedArchivedCloudWorkspace, queueTeardown: (providerSandboxId, delaySeconds) => queueTeardown( { providerSandboxId, driveName: null, attempt: 0 }, @@ -165,6 +254,16 @@ export interface RetentionRun { deferred: number; failed: number; }; + archived: + | { + found: number; + alreadyDeleted: number; + deleted: number; + skipped: number; + deferred: number; + failed: number; + } + | { skipped: "not production" }; orphans: | { listed: number; @@ -180,9 +279,10 @@ export interface RetentionRun { * The one place a workspace box is deleted for age. Every 6 hours: a workspace asleep * and unused for `RETENTION_IDLE_MS` fails as `snapshot_expired`, exactly as a * wake that met Vercel's expiry did, and its box is torn down. Then, in - * production only, boxes we tagged that no row names are torn down. Every - * deployment shares the Vercel project, so another database would see - * production's boxes as orphans. + * production only: an archived box still there a day after its grace is + * deleted (its own delete was lost), and boxes we tagged that no row names + * are torn down. Every deployment shares the Vercel project, and a dev + * database copied from production names production's boxes. */ export async function reapCloudWorkspaceRetention({ now = Date.now(), @@ -202,13 +302,22 @@ export async function reapCloudWorkspaceRetention({ deps.queueTeardown(name, slot++ * TEARDOWN_SPACING_SECONDS); const retiredNames = new Set(); const idle = await reapIdle({ now, deps, queue, retiredNames, outOfTime }); + const notProduction = { skipped: "not production" } as const; + const boxes = production ? await deps.listAllSandboxes() : []; + const archived = production + ? await reapArchived({ now, deps, boxes, outOfTime }) + : notProduction; const orphans = production - ? await reapOrphans({ now, deps, queue, retiredNames, outOfTime }) - : ({ skipped: "not production" } as const); - const run = { idle, orphans }; + ? await reapOrphans({ now, deps, boxes, queue, retiredNames, outOfTime }) + : notProduction; + const run = { idle, archived, orphans }; console.log("[cloud-workspace] retention run", JSON.stringify(run)); Sentry.logger.info("retention.run", flatten(run)); - if (idle.failed || ("failed" in orphans && orphans.failed)) { + if ( + idle.failed || + ("failed" in archived && archived.failed) || + ("failed" in orphans && orphans.failed) + ) { Sentry.captureMessage("cloud workspace retention run had failures", { level: "warning", extra: { ...run }, @@ -268,7 +377,61 @@ async function reapIdle({ return counts; } -function isOldAndSettled(box: TaggedSandbox, now: number): boolean { +async function reapArchived({ + now, + deps, + boxes, + outOfTime, +}: { + now: number; + deps: RetentionDeps; + boxes: ListedSandbox[]; + outOfTime: () => boolean; +}): Promise> { + const cutoff = new Date(now - ARCHIVED_BACKSTOP_AFTER_MS); + const found = await deps.countArchivedPastGrace(cutoff); + const lingering = await deps.findArchivedPastGraceAmong( + cutoff, + boxes.map((box) => box.name), + ); + const batch = lingering.slice(0, ARCHIVED_BATCH); + const counts = { + found, + alreadyDeleted: found - lingering.length, + deleted: 0, + skipped: 0, + deferred: lingering.length - batch.length, + failed: 0, + }; + for (const [index, row] of batch.entries()) { + if (outOfTime()) { + counts.deferred += batch.length - index; + break; + } + try { + const outcome = row.deletedAt + ? await deps.reapArchived({ + cloudWorkspaceId: row.id, + archivedAt: row.deletedAt.toISOString(), + stage: "delete", + }) + : await deps.reapUndatedArchived({ + cloudWorkspaceId: row.id, + archivedBefore: cutoff, + }); + if (outcome === "reaped") counts.deleted++; + else counts.skipped++; + } catch (error) { + counts.failed++; + Sentry.captureException(error, { + tags: { "cloud_workspace.id": row.id }, + }); + } + } + return counts; +} + +function isOldAndSettled(box: ListedSandbox, now: number): boolean { return ( SETTLED_STATUSES.has(box.status) && now - box.createdAt > ORPHAN_MIN_AGE_MS && @@ -279,21 +442,22 @@ function isOldAndSettled(box: TaggedSandbox, now: number): boolean { async function reapOrphans({ now, deps, + boxes, queue, retiredNames, outOfTime, }: { now: number; deps: RetentionDeps; + boxes: ListedSandbox[]; queue: (name: string) => Promise; retiredNames: Set; outOfTime: () => boolean; }): Promise> { - const boxes = [ - ...(await deps.listTaggedSandboxes("workspace")), - ...(await deps.listTaggedSandboxes("environment")), - ]; - const candidates = boxes + const ours = boxes.filter( + (box) => box.kind === "workspace" || box.kind === "environment", + ); + const candidates = ours .filter((box) => isOldAndSettled(box, now) && !retiredNames.has(box.name)) .map((box) => box.name); const named = await deps.namedSandboxes(candidates); @@ -316,7 +480,7 @@ async function reapOrphans({ } } return { - listed: boxes.length, + listed: ours.length, orphaned: orphaned.length, queued, deferred: orphaned.length - batch.length + unstarted, @@ -329,6 +493,12 @@ function flatten(run: RetentionRun): Record { ...Object.fromEntries( Object.entries(run.idle).map(([key, value]) => [`idle.${key}`, value]), ), + ...Object.fromEntries( + Object.entries(run.archived).map(([key, value]) => [ + `archived.${key}`, + value, + ]), + ), ...Object.fromEntries( Object.entries(run.orphans).map(([key, value]) => [ `orphans.${key}`, diff --git a/plans/done/20261118-1500-cloud-workspace-retention-reaper.md b/plans/done/20261118-1500-cloud-workspace-retention-reaper.md index 44753490493..ac0d7e76e54 100644 --- a/plans/done/20261118-1500-cloud-workspace-retention-reaper.md +++ b/plans/done/20261118-1500-cloud-workspace-retention-reaper.md @@ -81,6 +81,9 @@ None blocking. The QStash schedule must be created in the Upstash console by som - Decision: run every 6 hours (`0 */6 * * *`) with batches of 200 idle rows and 200 orphans, and stop starting work after 200 s of a run (`RUN_BUDGET_MS`). Rationale: Satya asked for 6-hourly runs sized to clear a normal day with room to spare. Production made a median 156 workspaces a day and at most 725 in the 30 days to 2026-11-18. Four runs of 200 is 800 a day, so even the peak day clears within one day, and a backlog takes several runs. The time budget keeps a large run inside the route's 300 s; the rest is reported as `deferred`. Date/Author: 2026-11-18, Satya via orchestrator. +- Decision: add an archived-rows backstop (orchestrator, required). With `snapshotExpiration: 0` a lost 7-day archive delete would keep a box and its snapshot forever. Each production run lists every box once. Archived rows past grace + 1 day whose box is in that listing are deleted through `reapArchivedCloudWorkspace({stage: "delete", archivedAt: deleted_at})`, or through `reapUndatedArchivedCloudWorkspace` (status archived, `deleted_at` null, `updated_at` older than the cutoff) for the 71 rows with no archive date. At most 20 a run, inside the same time budget, counted as found / alreadyDeleted / deleted / skipped / deferred / failed. + Rationale: matching against the listing covers every row each run, with no per-row provider read and no marker column; checking rows one by one would keep re-reading the same oldest rows. It is production only for the same reason as the orphan stage: a dev database branched from production names production's boxes. On 2026-11-18, a sample of 20 such boxes were all already gone, and 3,553 rows matched the condition. Date/Author: 2026-11-18, orchestrator. + ## Outcomes & Retrospective All seven parts of the brief are done except the drives. Workspace drives do not exist on main, so the drive half is a seam: `TeardownSandboxInput.driveName` is typed `null`, and #9847's version of `teardownSandbox` replaces this one at the same route with the same payload. The reaper's measure of last activity is deliberately later than Vercel's: the 412 rows whose `sandbox_expires_at` was moved forward on 2026-11-16 are reaped no earlier than 2026-12-16, and until then Vercel's own expiry and the 410 path still apply to them. From 72fa4bb7637b42a0a7c30d9745cc4ac346ccbddf Mon Sep 17 00:00:00 2001 From: Satya Patel <14907857+saddlepaddle@users.noreply.github.com> Date: Wed, 18 Nov 2026 23:20:00 +0000 Subject: [PATCH 4/8] refactor(cloud-workspace): retire idle boxes through retireAndTearDown, #9847's seam The idle stage takes the box name off the row with retireAndTearDown(row, condition), the name, signature and module #9847 uses, then fails the row. Every box, with a drive or not, goes through the retried teardown job, and the run paces teardowns in its loop since the seam takes no delay. --- .../api/cloud-workspaces/teardown/route.ts | 11 ++-- docs/cloud-sandbox-mismatches.md | 5 +- .../router/cloud-workspace/retention.test.ts | 64 ++++++++++++------- .../src/router/cloud-workspace/retention.ts | 63 ++++++++---------- .../router/cloud-workspace/teardown.test.ts | 55 +++++++++++++++- .../src/router/cloud-workspace/teardown.ts | 57 +++++++++++++++-- ...8-1500-cloud-workspace-retention-reaper.md | 13 ++-- 7 files changed, 190 insertions(+), 78 deletions(-) diff --git a/apps/api/src/app/api/cloud-workspaces/teardown/route.ts b/apps/api/src/app/api/cloud-workspaces/teardown/route.ts index 46ec2aa7a4f..f446716d52c 100644 --- a/apps/api/src/app/api/cloud-workspaces/teardown/route.ts +++ b/apps/api/src/app/api/cloud-workspaces/teardown/route.ts @@ -1,7 +1,4 @@ -import { - TEARDOWN_JOB_PATH, - teardownSandbox, -} from "@superset/trpc/cloud-workspace-teardown"; +import { teardownSandbox } from "@superset/trpc/cloud-workspace-teardown"; import { z } from "zod"; import { verifyQstashRequest } from "@/lib/verifyQstash"; @@ -19,7 +16,11 @@ const payloadSchema = z /** Stops a box, then deletes it with its snapshots; requeues itself on failure. */ export async function POST(request: Request): Promise { const body = await request.text(); - const rejected = await verifyQstashRequest(request, body, TEARDOWN_JOB_PATH); + const rejected = await verifyQstashRequest( + request, + body, + "/api/cloud-workspaces/teardown", + ); if (rejected) return rejected; const parsed = payloadSchema.safeParse(JSON.parse(body)); diff --git a/docs/cloud-sandbox-mismatches.md b/docs/cloud-sandbox-mismatches.md index ab68f3cf9f7..517b7bf2de6 100644 --- a/docs/cloud-sandbox-mismatches.md +++ b/docs/cloud-sandbox-mismatches.md @@ -792,8 +792,9 @@ could not resume, and the box itself stayed in Vercel. New workspace boxes are now created with `snapshotExpiration: 0`. The `reap-retention` job, every 6 hours, (`router/cloud-workspace/retention.ts`) does three things. It moves a workspace that is asleep and unused for 30 days to `failed`/`snapshot_expired`, the same -state as the 410. It gives the row a new box name in that same update. Then it -queues `teardownSandbox`, which deletes the box only if no row names it. "Unused" +state as the 410. First `retireAndTearDown` gives the row a new box name and +queues `teardownSandbox`, which deletes the box only if no row names it; then +the row fails. A wake between the two finds no box and fails the row itself. "Unused" means no boot started or stopped since then, with `sandbox_expires_at` and `created_at` older too, so a wake in flight (its boot row is written before Vercel resumes) keeps the box. Archive deletes its box at the end of the diff --git a/packages/trpc/src/router/cloud-workspace/retention.test.ts b/packages/trpc/src/router/cloud-workspace/retention.test.ts index 5f5aaa15fe4..8a6bfe1aef4 100644 --- a/packages/trpc/src/router/cloud-workspace/retention.test.ts +++ b/packages/trpc/src/router/cloud-workspace/retention.test.ts @@ -9,7 +9,9 @@ import { RUN_BUDGET_MS, reapCloudWorkspaceRetention, retireIdleWorkspace, + TEARDOWN_SPACING_MS, } from "./retention"; +import * as teardown from "./teardown"; import * as transition from "./transition"; const NOW = Date.parse("2026-11-18T06:00:00Z"); @@ -34,7 +36,8 @@ const box = ( }); function fakeDeps(overrides: Partial = {}) { - const queued: Array<[string, number]> = []; + const queued: string[] = []; + const paused: number[] = []; const retired: string[] = []; const reaped: unknown[] = []; const deps: RetentionDeps = { @@ -56,18 +59,21 @@ function fakeDeps(overrides: Partial = {}) { reaped.push(input); return "reaped"; }, - queueTeardown: async (name, delaySeconds) => { - queued.push([name, delaySeconds]); + tearDownOrphan: async (name) => { + queued.push(name); + }, + pause: async (ms) => { + paused.push(ms); }, ...overrides, }; - return { deps, queued, retired, reaped }; + return { deps, queued, paused, retired, reaped }; } describe("reapCloudWorkspaceRetention", () => { - test("retires each idle workspace and queues its box's teardown a second apart", async () => { + test("retires each idle workspace, a pause apart", async () => { let cutoff: Date | undefined; - const { deps, queued, retired } = fakeDeps({ + const { deps, paused, retired } = fakeDeps({ findIdleWorkspaces: async (at) => { cutoff = at; return [idleRow(1), idleRow(2)]; @@ -80,15 +86,12 @@ describe("reapCloudWorkspaceRetention", () => { }); expect(cutoff?.getTime()).toBe(NOW - RETENTION_IDLE_MS); expect(retired).toEqual(["ws_1", "ws_2"]); - expect(queued).toEqual([ - ["ws-box1", 0], - ["ws-box2", 1], - ]); + expect(paused).toEqual([TEARDOWN_SPACING_MS, TEARDOWN_SPACING_MS]); expect(run.idle).toMatchObject({ found: 2, retired: 2 }); }); test("a box that runs a session now is skipped, not retired", async () => { - const { deps, queued, retired } = fakeDeps({ + const { deps, retired } = fakeDeps({ findIdleWorkspaces: async () => [idleRow(1)], boxIsRunning: async () => true, }); @@ -98,12 +101,11 @@ describe("reapCloudWorkspaceRetention", () => { deps, }); expect(retired).toEqual([]); - expect(queued).toEqual([]); expect(run.idle.skippedRunning).toBe(1); }); test("a row woken or changed after it was read keeps its box", async () => { - const { deps, queued } = fakeDeps({ + const { deps, paused } = fakeDeps({ findIdleWorkspaces: async () => [idleRow(1)], retireIdleWorkspace: async () => false, }); @@ -112,7 +114,7 @@ describe("reapCloudWorkspaceRetention", () => { production: false, deps, }); - expect(queued).toEqual([]); + expect(paused).toEqual([]); expect(run.idle.skippedChanged).toBe(1); }); @@ -136,11 +138,7 @@ describe("reapCloudWorkspaceRetention", () => { production: true, deps, }); - expect(queued).toEqual([ - ["ws-box1", 0], - ["ws-orphan", 1], - ["env-orphan", 2], - ]); + expect(queued).toEqual(["ws-orphan", "env-orphan"]); expect(run.orphans).toEqual({ listed: 7, orphaned: 2, @@ -270,7 +268,17 @@ describe("reapCloudWorkspaceRetention", () => { }); describe("retireIdleWorkspace", () => { + let retiring: Parameters | undefined; + let retires = true; let failed: Parameters[0] | undefined; + stub(teardown, { + retireAndTearDown: async ( + ...args: Parameters + ) => { + retiring = args; + return retires; + }, + }); stub(transition, { failCloudWorkspace: async ( args: Parameters[0], @@ -280,8 +288,19 @@ describe("retireIdleWorkspace", () => { }, }); - test("fails the row as an expired snapshot and gives it a new box name in the same update", async () => { - await retireIdleWorkspace(idleRow(1), new Date(NOW - RETENTION_IDLE_MS)); + test("retires the box name while the row is asleep and idle, then fails the row as an expired snapshot", async () => { + const cutoff = new Date(NOW - RETENTION_IDLE_MS); + retires = false; + expect(await retireIdleWorkspace(idleRow(1), cutoff)).toBe(false); + expect(failed).toBeUndefined(); + retires = true; + expect(await retireIdleWorkspace(idleRow(1), cutoff)).toBe(true); + expect(retiring?.[0]).toEqual({ + id: "ws_1", + providerSandboxId: "ws-box1", + driveName: null, + }); + expect(retiring?.[1]).toBeDefined(); expect(failed).toMatchObject({ scope: { cloudWorkspaceId: "ws_1", organizationId: "org_1" }, from: ["asleep"], @@ -289,8 +308,5 @@ describe("retireIdleWorkspace", () => { expected: true, set: { sandboxUrl: null }, }); - expect(failed?.set?.providerSandboxId).toMatch(/^ws-ws_1[0-9a-f]{8}$/); - expect(failed?.set?.providerSandboxId).not.toBe("ws-box1"); - expect(failed?.where).toBeDefined(); }); }); diff --git a/packages/trpc/src/router/cloud-workspace/retention.ts b/packages/trpc/src/router/cloud-workspace/retention.ts index f1dc71b6b4a..72b63542846 100644 --- a/packages/trpc/src/router/cloud-workspace/retention.ts +++ b/packages/trpc/src/router/cloud-workspace/retention.ts @@ -20,13 +20,12 @@ import { listAllSandboxes, SandboxUnavailableError, } from "../../lib/sandbox"; -import { nextSandboxNameFor } from "./provision"; import { ARCHIVE_GRACE_SECONDS, reapArchivedCloudWorkspace, reapUndatedArchivedCloudWorkspace, } from "./reap"; -import { queueTeardown } from "./teardown"; +import { deleteSandboxAndDriveLater, retireAndTearDown } from "./teardown"; import { namedSandboxes } from "./teardown-guard"; import { failCloudWorkspace } from "./transition"; @@ -49,8 +48,8 @@ export const ARCHIVED_BATCH = 20; export const ARCHIVED_BACKSTOP_AFTER_MS = ARCHIVE_GRACE_SECONDS * 1000 + DAY_MS; /** Stops starting new work here, well inside the route's 300 s; the rest waits for the next run. */ export const RUN_BUDGET_MS = 200_000; -/** One delete a second: far under Vercel's 20 deletions a second per team. */ -const TEARDOWN_SPACING_SECONDS = 1; +/** At most 4 teardowns queued a second: far under Vercel's 20 deletions a second per team. */ +export const TEARDOWN_SPACING_MS = 250; const NAME_LOOKUP_CHUNK = 500; const SETTLED_STATUSES = new Set(["stopped", "failed", "aborted"]); @@ -105,29 +104,30 @@ async function findIdleWorkspaces( } /** - * The 410 path's terminal state, and the box's name taken off the row in the - * same update, so nothing can wake the box being deleted. False when the row + * Takes the box's name off the row while it is still asleep and idle, and + * queues the box's teardown; then fails the row as the 410 path does. A wake + * between the two finds no box and fails the row itself. False when the row * changed since it was read: woken, archived, or given another box. */ -export function retireIdleWorkspace( +export async function retireIdleWorkspace( row: IdleWorkspace, cutoff: Date, ): Promise { - return failCloudWorkspace({ + const retired = await retireAndTearDown( + { id: row.id, providerSandboxId: row.providerSandboxId, driveName: null }, + and(eq(cloudWorkspaces.status, "asleep"), stillIdleAndAsleep(cutoff)), + ); + if (!retired) return false; + await failCloudWorkspace({ scope: { cloudWorkspaceId: row.id, organizationId: row.organizationId }, from: ["asleep"], - where: and( - eq(cloudWorkspaces.providerSandboxId, row.providerSandboxId), - stillIdleAndAsleep(cutoff), - ), - set: { - sandboxUrl: null, - providerSandboxId: nextSandboxNameFor(row.id), - }, + where: stillIdleAndAsleep(cutoff), + set: { sandboxUrl: null }, reason: "snapshot_expired", error: new Error(`idle since before ${cutoff.toISOString()}`), expected: true, }); + return true; } /** True when the box runs a session now; false when it is stopped or gone. */ @@ -222,10 +222,8 @@ export interface RetentionDeps { findArchivedPastGraceAmong: typeof findArchivedPastGraceAmong; reapArchived: typeof reapArchivedCloudWorkspace; reapUndatedArchived: typeof reapUndatedArchivedCloudWorkspace; - queueTeardown: ( - providerSandboxId: string, - delaySeconds: number, - ) => Promise; + tearDownOrphan: (providerSandboxId: string) => Promise; + pause: (ms: number) => Promise; } const defaultDeps: RetentionDeps = { @@ -238,11 +236,9 @@ const defaultDeps: RetentionDeps = { findArchivedPastGraceAmong, reapArchived: reapArchivedCloudWorkspace, reapUndatedArchived: reapUndatedArchivedCloudWorkspace, - queueTeardown: (providerSandboxId, delaySeconds) => - queueTeardown( - { providerSandboxId, driveName: null, attempt: 0 }, - delaySeconds, - ), + tearDownOrphan: (providerSandboxId) => + deleteSandboxAndDriveLater({ providerSandboxId, driveName: null }), + pause: (ms) => new Promise((resolve) => setTimeout(resolve, ms)), }; export interface RetentionRun { @@ -297,18 +293,15 @@ export async function reapCloudWorkspaceRetention({ }): Promise { const startedAt = clock(); const outOfTime = () => clock() - startedAt > RUN_BUDGET_MS; - let slot = 0; - const queue = (name: string) => - deps.queueTeardown(name, slot++ * TEARDOWN_SPACING_SECONDS); const retiredNames = new Set(); - const idle = await reapIdle({ now, deps, queue, retiredNames, outOfTime }); + const idle = await reapIdle({ now, deps, retiredNames, outOfTime }); const notProduction = { skipped: "not production" } as const; const boxes = production ? await deps.listAllSandboxes() : []; const archived = production ? await reapArchived({ now, deps, boxes, outOfTime }) : notProduction; const orphans = production - ? await reapOrphans({ now, deps, boxes, queue, retiredNames, outOfTime }) + ? await reapOrphans({ now, deps, boxes, retiredNames, outOfTime }) : notProduction; const run = { idle, archived, orphans }; console.log("[cloud-workspace] retention run", JSON.stringify(run)); @@ -329,13 +322,11 @@ export async function reapCloudWorkspaceRetention({ async function reapIdle({ now, deps, - queue, retiredNames, outOfTime, }: { now: number; deps: RetentionDeps; - queue: (name: string) => Promise; retiredNames: Set; outOfTime: () => boolean; }): Promise { @@ -365,8 +356,7 @@ async function reapIdle({ } counts.retired++; retiredNames.add(row.providerSandboxId); - // A retired box whose teardown is not queued is an orphan: a later run queues it. - await queue(row.providerSandboxId); + await deps.pause(TEARDOWN_SPACING_MS); } catch (error) { counts.failed++; Sentry.captureException(error, { @@ -443,14 +433,12 @@ async function reapOrphans({ now, deps, boxes, - queue, retiredNames, outOfTime, }: { now: number; deps: RetentionDeps; boxes: ListedSandbox[]; - queue: (name: string) => Promise; retiredNames: Set; outOfTime: () => boolean; }): Promise> { @@ -472,8 +460,9 @@ async function reapOrphans({ break; } try { - await queue(name); + await deps.tearDownOrphan(name); queued++; + await deps.pause(TEARDOWN_SPACING_MS); } catch (error) { failed++; Sentry.captureException(error, { extra: { providerSandboxId: name } }); diff --git a/packages/trpc/src/router/cloud-workspace/teardown.test.ts b/packages/trpc/src/router/cloud-workspace/teardown.test.ts index fa8711bd488..a7e543cd9f3 100644 --- a/packages/trpc/src/router/cloud-workspace/teardown.test.ts +++ b/packages/trpc/src/router/cloud-workspace/teardown.test.ts @@ -1,10 +1,12 @@ import { beforeEach, describe, expect, test } from "bun:test"; +import { db } from "@superset/db/client"; import { stub } from "../../../test/stub"; import * as sandbox from "../../lib/sandbox"; import * as jobs from "./jobs"; import { + deleteSandboxAndDriveLater, + retireAndTearDown, TEARDOWN_ATTEMPTS, - TEARDOWN_JOB_PATH, teardownSandbox, } from "./teardown"; import * as guard from "./teardown-guard"; @@ -13,6 +15,8 @@ let calls: string[] = []; let namedOnRead: boolean[] = []; let deleteFails = false; let queued: Array<{ path: string; body: unknown; delaySeconds?: number }> = []; +let updateMatches = true; +let renamedTo: unknown; stub(guard, { liveReferences: async () => ({ sandbox: namedOnRead.shift() ?? false }), @@ -27,6 +31,18 @@ stub(sandbox, { calls.push(`delete:${name}`); }, }); +stub(db, { + update: () => ({ + set: (values: { providerSandboxId: string }) => { + renamedTo = values.providerSandboxId; + return { + where: () => ({ + returning: async () => (updateMatches ? [{ id: "ws_1" }] : []), + }), + }; + }, + }), +}); stub(jobs, { publishCloudWorkspaceJob: async (job: { path: string; @@ -42,6 +58,8 @@ beforeEach(() => { namedOnRead = []; deleteFails = false; queued = []; + updateMatches = true; + renamedTo = undefined; }); const input = { providerSandboxId: "ws-old", driveName: null, attempt: 0 }; @@ -62,7 +80,10 @@ describe("teardownSandbox", () => { deleteFails = true; expect(await teardownSandbox(input)).toBe("retrying"); expect(queued).toMatchObject([ - { path: TEARDOWN_JOB_PATH, body: { ...input, attempt: 1 } }, + { + path: "/api/cloud-workspaces/teardown", + body: { ...input, attempt: 1 }, + }, ]); queued = []; expect( @@ -71,3 +92,33 @@ describe("teardownSandbox", () => { expect(queued).toEqual([]); }); }); + +describe("retireAndTearDown", () => { + const row = { id: "ws_1", providerSandboxId: "ws-old", driveName: null }; + + test("gives the row a new box name and queues the old box's teardown only when the update matched", async () => { + updateMatches = false; + expect(await retireAndTearDown(row, undefined)).toBe(false); + expect(queued).toEqual([]); + updateMatches = true; + expect(await retireAndTearDown(row, undefined)).toBe(true); + expect(renamedTo).toMatch(/^ws-ws_1[0-9a-f]{8}$/); + expect(queued).toMatchObject([ + { path: "/api/cloud-workspaces/teardown", body: input }, + ]); + expect(calls).toEqual([]); + }); +}); + +describe("deleteSandboxAndDriveLater", () => { + test("a box without a drive goes through the retried job too, not an inline delete", async () => { + await deleteSandboxAndDriveLater({ + providerSandboxId: "ws-old", + driveName: null, + }); + expect(calls).toEqual([]); + expect(queued).toMatchObject([ + { path: "/api/cloud-workspaces/teardown", body: input }, + ]); + }); +}); diff --git a/packages/trpc/src/router/cloud-workspace/teardown.ts b/packages/trpc/src/router/cloud-workspace/teardown.ts index 25229e239f4..0ce81e9ebbd 100644 --- a/packages/trpc/src/router/cloud-workspace/teardown.ts +++ b/packages/trpc/src/router/cloud-workspace/teardown.ts @@ -1,10 +1,12 @@ import * as Sentry from "@sentry/core"; +import { db } from "@superset/db/client"; +import { cloudWorkspaces } from "@superset/db/schema"; +import { and, eq, type SQL } from "drizzle-orm"; import { deleteSandbox, stopSandbox } from "../../lib/sandbox"; import { publishCloudWorkspaceJob } from "./jobs"; +import { nextSandboxNameFor } from "./provision"; import { liveReferences } from "./teardown-guard"; -export const TEARDOWN_JOB_PATH = "/api/cloud-workspaces/teardown"; - export interface TeardownSandboxInput { providerSandboxId: string; /** Always null until workspace drives exist; #9847 tears down the drive here. */ @@ -69,14 +71,61 @@ async function retryOrAbandon( return "retrying"; } -export function queueTeardown( +function queueTeardown( input: TeardownSandboxInput, delaySeconds?: number, ): Promise { return publishCloudWorkspaceJob({ - path: TEARDOWN_JOB_PATH, + path: "/api/cloud-workspaces/teardown", body: input, delaySeconds, runLocally: teardownSandbox, }); } + +/** Every box goes through the retried job, which reads the rows again just before it deletes. */ +export async function deleteSandboxAndDriveLater(args: { + providerSandboxId: string; + driveName: string | null | undefined; + keepDrive?: boolean; +}): Promise { + if (args.driveName) { + throw new Error( + `no drive teardown yet: ${args.providerSandboxId} names drive ${args.driveName}`, + ); + } + await queueTeardown({ + providerSandboxId: args.providerSandboxId, + driveName: null, + attempt: 0, + }); +} + +/** + * Takes the row's box name out of use in the same write that commits to + * deleting the box, then queues its teardown. A wake after it finds a name + * that does not exist, never the box being deleted. Returns false when the + * row no longer matches `condition` or names another box. + */ +export async function retireAndTearDown( + row: { id: string; providerSandboxId: string; driveName: string | null }, + condition: SQL | undefined, +): Promise { + const [retired] = await db + .update(cloudWorkspaces) + .set({ providerSandboxId: nextSandboxNameFor(row.id) }) + .where( + and( + eq(cloudWorkspaces.id, row.id), + eq(cloudWorkspaces.providerSandboxId, row.providerSandboxId), + condition, + ), + ) + .returning({ id: cloudWorkspaces.id }); + if (!retired) return false; + await deleteSandboxAndDriveLater({ + providerSandboxId: row.providerSandboxId, + driveName: row.driveName, + }); + return true; +} diff --git a/plans/done/20261118-1500-cloud-workspace-retention-reaper.md b/plans/done/20261118-1500-cloud-workspace-retention-reaper.md index ac0d7e76e54..15661df5b80 100644 --- a/plans/done/20261118-1500-cloud-workspace-retention-reaper.md +++ b/plans/done/20261118-1500-cloud-workspace-retention-reaper.md @@ -8,7 +8,7 @@ Reference: This plan follows conventions from AGENTS.md and the ExecPlan templat A cloud workspace is a row in `cloud_workspaces` backed by a Vercel sandbox ("box"). When the box stops, Vercel keeps a snapshot of its disk, and the next wake resumes from it. Today `provisionSandbox` in `packages/trpc/src/lib/sandbox/vercel.ts` creates workspace boxes with `snapshotExpiration: 30 days`, so Vercel deletes the snapshot of a box nobody used for 30 days. The app learns this only when a wake, an agent turn or an automation dispatch gets a 410 and calls `markSandboxUnavailable` (`router/cloud-workspace/wake.ts`), which moves the row to `failed` with reason `snapshot_expired`. A workspace nobody opens again never learns. The box itself, and boxes that no row names (orphans), stay in Vercel and use storage. -After this change Vercel never expires a new workspace's snapshot. A job that runs every 6 hours decides instead. It moves a workspace that has been asleep with no activity for 30 days to the same `failed`/`snapshot_expired` state the 410 path gives, takes the box name off the row in the same update, and queues a retried teardown that deletes the box. The same job queues teardown for orphan boxes. Each run logs and returns how many it retired, queued and skipped, and a Sentry cron monitor reports a run that did not happen. The customer sees the same 30-day retention as today. +After this change Vercel never expires a new workspace's snapshot. A job that runs every 6 hours decides instead. It moves a workspace that has been asleep with no activity for 30 days to the same `failed`/`snapshot_expired` state the 410 path gives, takes the box name off the row first (through `retireAndTearDown`, the function #9847 owns), and queues a retried teardown that deletes the box. The same job queues teardown for orphan boxes. Each run logs and returns how many it retired, queued and skipped, and a Sentry cron monitor reports a run that did not happen. The customer sees the same 30-day retention as today. ## Context and Orientation @@ -25,12 +25,12 @@ Last activity. `sandbox_expires_at` on an asleep row is when its last session en ## Plan of Work 1. `vercel.ts`: workspace boxes are created with `snapshotExpiration: 0` like goldens. Remove `WORKSPACE_SNAPSHOT_EXPIRATION_MS`. Add `listTaggedSandboxes(kind)`, which lists boxes of one tag kind with `createdAt`, `statusUpdatedAt`, `status` and `snapshotExpiration`. -2. `router/cloud-workspace/teardown.ts` (new): `teardownSandbox(input)` checks `liveReferences` (any workspace row in any status naming the box, or an unarchived environment's `source_ref`), stops the box, checks again, deletes it (`deleteSandbox`, which also deletes its snapshots). On error it requeues itself up to `TEARDOWN_ATTEMPTS` times. `queueTeardown(input, delaySeconds)`. `retireAndTearDown` is left to #9847; the reaper does its own atomic retire because it also changes the status. +2. `router/cloud-workspace/teardown.ts` (new): `teardownSandbox(input)` checks `liveReferences` (any workspace row in any status naming the box, or an unarchived environment's `source_ref`), stops the box, checks again, deletes it (`deleteSandbox`, which also deletes its snapshots). On error it requeues itself up to `TEARDOWN_ATTEMPTS` times. `deleteSandboxAndDriveLater({providerSandboxId, driveName})` queues that job for every box (box-only: a drive name throws). `retireAndTearDown(row, condition)` is one UPDATE WHERE id AND provider_sandbox_id AND condition that sets a new box name; only if it matched, it calls `deleteSandboxAndDriveLater`. Same names, signatures and module path as #9847, so #9847's file replaces this one. 3. `transition.ts`: `failCloudWorkspace` takes an optional `where` (extra conditions on the same update) and `expected` (log a warning, no Sentry exception), and returns whether it moved the row. Existing callers are unchanged. 4. `router/cloud-workspace/retention.ts` (new): `reapIdleCloudWorkspaces({now, production, deps})`. - - Idle stage: select up to `REAP_BATCH` asleep vercel rows idle since `now - 30d`. For each, a provider read skips a box that is running. Then one `failCloudWorkspace` update from `asleep`, with `where` = same name + still idle + not archived, sets `failure_reason = snapshot_expired`, `sandbox_url = null` and `provider_sandbox_id = nextSandboxNameFor(id)`. Only if it moved, queue the teardown of the old name. + - Idle stage: select up to `REAP_BATCH` asleep vercel rows idle since `now - 30d`. For each, a provider read skips a box that is running. Then `retireAndTearDown(row, status = asleep AND still idle AND not archived)` takes the name off the row and queues the old box's teardown. Only if it matched, `failCloudWorkspace` from `asleep` (still idle) sets `failure_reason = snapshot_expired` and `sandbox_url = null`. A wake between the two writes finds no box and fails the row itself, so the second write then moves nothing. - Orphan stage (only when `production`): list `workspace` and `environment` tagged boxes. Keep those in a settled status whose `createdAt` and `statusUpdatedAt` are older than `ORPHAN_MIN_AGE` (7 days). Drop every name a workspace row or an unarchived environment names, and queue teardown for up to `ORPHAN_BATCH`. - - Teardowns are staggered one per second (`delaySeconds = index`), far below Vercel's 20 deletions/s per team. + - The run pauses `TEARDOWN_SPACING_MS` (250 ms) after each teardown it queues, so at most 4 a second, far below Vercel's 20 deletions/s per team. `retireAndTearDown` takes no delay, so the pacing is in the loop. - It returns counts (`retired`, `orphansQueued`, `skipped{running,raced,orphanBatchFull,…}`) and logs them as one line. 5. `apps/api/.../cloud-workspaces/jobs/reap-retention/route.ts` (every 6 hours, `Sentry.withMonitor("reap-retention", …, "0 */6 * * *")`) and `.../cloud-workspaces/teardown/route.ts`. Add `reap-retention` to the publish workflow's choices. Add package exports. 6. `packages/sandbox/src/backfill-snapshot-expiration.ts` (new, not run): a dry run by default. `--apply` calls `sandbox.update({snapshotExpiration: 0})` on workspace boxes whose `snapshotExpiration` is not 0, at 2/s. @@ -84,6 +84,11 @@ None blocking. The QStash schedule must be created in the Upstash console by som - Decision: add an archived-rows backstop (orchestrator, required). With `snapshotExpiration: 0` a lost 7-day archive delete would keep a box and its snapshot forever. Each production run lists every box once. Archived rows past grace + 1 day whose box is in that listing are deleted through `reapArchivedCloudWorkspace({stage: "delete", archivedAt: deleted_at})`, or through `reapUndatedArchivedCloudWorkspace` (status archived, `deleted_at` null, `updated_at` older than the cutoff) for the 71 rows with no archive date. At most 20 a run, inside the same time budget, counted as found / alreadyDeleted / deleted / skipped / deferred / failed. Rationale: matching against the listing covers every row each run, with no per-row provider read and no marker column; checking rows one by one would keep re-reading the same oldest rows. It is production only for the same reason as the orphan stage: a dev database branched from production names production's boxes. On 2026-11-18, a sample of 20 such boxes were all already gone, and 3,553 rows matched the condition. Date/Author: 2026-11-18, orchestrator. +- Decision: the idle stage retires names through `retireAndTearDown(row, condition)` with #9847's exact name, signature and module, and every box (with or without a drive) goes through the retried teardown job (orchestrator). + Rationale: #9847's file then replaces this one with no change to `retention.ts`. The cost: the name and the status are now two writes, not one. The name goes first, so the window is an asleep row that names a box that does not exist, which the 410 path already handles. The other order would leave a failed row naming a box being deleted. `deleteSandboxAndDriveLater` on #9847 deletes a box without a drive inline, with no retry and no re-read; this PR queues it, because the orphan stage depends on the re-read just before delete. Date/Author: 2026-11-18, orchestrator. + ## Outcomes & Retrospective All seven parts of the brief are done except the drives. Workspace drives do not exist on main, so the drive half is a seam: `TeardownSandboxInput.driveName` is typed `null`, and #9847's version of `teardownSandbox` replaces this one at the same route with the same payload. The reaper's measure of last activity is deliberately later than Vercel's: the 412 rows whose `sandbox_expires_at` was moved forward on 2026-11-16 are reaped no earlier than 2026-12-16, and until then Vercel's own expiry and the 410 path still apply to them. + +Revision note (2026-11-18): aligned with #9847's `retireAndTearDown` and `deleteSandboxAndDriveLater`; the idle stage's single update became two writes, and teardowns are paced in the loop instead of by `delaySeconds`. From 200df345a4a0b043589c77885bd4d44962f2b122 Mon Sep 17 00:00:00 2001 From: Satya Patel <14907857+saddlepaddle@users.noreply.github.com> Date: Thu, 19 Nov 2026 00:23:26 +0000 Subject: [PATCH 5/8] feat(cloud-workspace): retire the boxes of failed and stranded provisioning rows The retention run now also takes the box from a row failed 30 days ago whose box is still listed, and from a row stuck provisioning for a day, through retireAndTearDown with the status and age it read. A stranded row then fails. Retired rows leave the selection, so none can fill the batch forever. This is the box half of the drive-liveness property #9914's model found failing; the drive half waits for #9847's drive_name. --- docs/cloud-sandbox-mismatches.md | 6 +- .../router/cloud-workspace/retention.test.ts | 140 ++++++++++++ .../src/router/cloud-workspace/retention.ts | 206 +++++++++++++++++- ...8-1500-cloud-workspace-retention-reaper.md | 5 + 4 files changed, 353 insertions(+), 4 deletions(-) diff --git a/docs/cloud-sandbox-mismatches.md b/docs/cloud-sandbox-mismatches.md index 517b7bf2de6..9b12879a997 100644 --- a/docs/cloud-sandbox-mismatches.md +++ b/docs/cloud-sandbox-mismatches.md @@ -803,7 +803,11 @@ to clean up after a lost one. That backstop is gone, so in production the job also deletes an archived box still listed a day after its grace, through the archive's own delete stage (`reapUndatedArchivedCloudWorkspace` for the rows archived before `deleted_at` existed). It matches rows against one listing of -every box, so a box already gone costs no provider call. In production only, +every box, so a box already gone costs no provider call. It also takes the box +from a row failed 30 days ago whose box is still listed, and from a row stuck +in `provisioning` for a day (every delivery of its job died), which then fails +as `create_failed`. A retired name is in no listing and a failed row is no +longer provisioning, so these rows leave the selection. In production only, the same job also tears down tagged boxes that no row and no unarchived environment names, once they are 7 days old: every deployment shares one Vercel project, so a dev or preview diff --git a/packages/trpc/src/router/cloud-workspace/retention.test.ts b/packages/trpc/src/router/cloud-workspace/retention.test.ts index 8a6bfe1aef4..ed0e34d2523 100644 --- a/packages/trpc/src/router/cloud-workspace/retention.test.ts +++ b/packages/trpc/src/router/cloud-workspace/retention.test.ts @@ -1,14 +1,20 @@ import { describe, expect, test } from "bun:test"; +import { db } from "@superset/db/client"; import { stub } from "../../../test/stub"; import type { ListedSandbox } from "../../lib/sandbox"; import { ARCHIVED_BACKSTOP_AFTER_MS, + findStranded, ORPHAN_BATCH, RETENTION_IDLE_MS, type RetentionDeps, RUN_BUDGET_MS, reapCloudWorkspaceRetention, retireIdleWorkspace, + retireStrandedWorkspace, + STRANDED_BATCH, + STRANDED_PROVISIONING_MS, + type StrandedWorkspace, TEARDOWN_SPACING_MS, } from "./retention"; import * as teardown from "./teardown"; @@ -23,6 +29,17 @@ const idleRow = (n: number) => ({ providerSandboxId: `ws-box${n}`, }); +const strandedRow = ( + id: string, + providerSandboxId: string, + status: StrandedWorkspace["status"], +): StrandedWorkspace => ({ + id, + organizationId: "org_1", + providerSandboxId, + status, +}); + const box = ( name: string, overrides: Partial = {}, @@ -59,6 +76,11 @@ function fakeDeps(overrides: Partial = {}) { reaped.push(input); return "reaped"; }, + findStranded: async () => [], + retireStrandedWorkspace: async (row) => { + retired.push(row.id); + return true; + }, tearDownOrphan: async (name) => { queued.push(name); }, @@ -167,9 +189,68 @@ describe("reapCloudWorkspaceRetention", () => { expect(listed).toBe(false); expect(reaped).toEqual([]); expect(run.archived).toEqual({ skipped: "not production" }); + expect(run.stranded).toEqual({ skipped: "not production" }); expect(run.orphans).toEqual({ skipped: "not production" }); }); + test("retires failed and stranded rows that still name a box, a pause apart, and does not queue their boxes again as orphans", async () => { + let asked: Parameters[0] | undefined; + const { deps, retired, queued, paused } = fakeDeps({ + listAllSandboxes: async () => [box("ws-failed"), box("ws-stranded")], + findStranded: async (args) => { + asked = args; + return [ + strandedRow("ws_f", "ws-failed", "failed"), + strandedRow("ws_p", "ws-stranded", "provisioning"), + ]; + }, + }); + const run = await reapCloudWorkspaceRetention({ + now: NOW, + production: true, + deps, + }); + expect(asked).toEqual({ + failedCutoff: new Date(NOW - RETENTION_IDLE_MS), + provisioningCutoff: new Date(NOW - STRANDED_PROVISIONING_MS), + names: ["ws-failed", "ws-stranded"], + limit: STRANDED_BATCH, + }); + expect(retired).toEqual(["ws_f", "ws_p"]); + expect(paused).toEqual([TEARDOWN_SPACING_MS, TEARDOWN_SPACING_MS]); + expect(queued).toEqual([]); + expect(run.stranded).toEqual({ + found: 2, + retired: 2, + skippedChanged: 0, + deferred: 0, + failed: 0, + }); + }); + + test("a failed or stranded row that changed or could not be retired is counted, and the next run tries again", async () => { + const { deps } = fakeDeps({ + findStranded: async () => [ + strandedRow("ws_f", "ws-failed", "failed"), + strandedRow("ws_p", "ws-stranded", "provisioning"), + ], + retireStrandedWorkspace: async (row) => { + if (row.status === "failed") return false; + throw new Error("db down"); + }, + }); + const run = await reapCloudWorkspaceRetention({ + now: NOW, + production: true, + deps, + }); + expect(run.stranded).toMatchObject({ + retired: 0, + skippedChanged: 1, + failed: 1, + }); + }); + test("deletes an archived box still there past its grace through the archive's own delete, by the guard that fits the row", async () => { let cutoff: Date | undefined; let askedAbout: string[] = []; @@ -309,4 +390,63 @@ describe("retireIdleWorkspace", () => { set: { sandboxUrl: null }, }); }); + + test("a stranded provisioning row gives up its box and fails; a failed row only gives up its box", async () => { + const cutoffs = { + failed: new Date(NOW - RETENTION_IDLE_MS), + provisioning: new Date(NOW - STRANDED_PROVISIONING_MS), + }; + retires = true; + failed = undefined; + expect( + await retireStrandedWorkspace( + strandedRow("ws_f", "ws-failed", "failed"), + cutoffs, + ), + ).toBe(true); + expect(retiring?.[0]).toEqual({ + id: "ws_f", + providerSandboxId: "ws-failed", + driveName: null, + }); + expect(failed).toBeUndefined(); + expect( + await retireStrandedWorkspace( + strandedRow("ws_p", "ws-stranded", "provisioning"), + cutoffs, + ), + ).toBe(true); + expect(failed).toMatchObject({ + scope: { cloudWorkspaceId: "ws_p" }, + from: ["provisioning"], + reason: "create_failed", + expected: true, + }); + }); +}); + +describe("findStranded", () => { + let queries = 0; + const chain = { + from: () => chain, + where: () => chain, + orderBy: () => chain, + limit: async () => { + queries++; + return []; + }, + }; + stub(db, { select: () => chain }); + + test("looks for failed rows only among listed boxes, so a retired name leaves the selection", async () => { + const args = { + failedCutoff: new Date(NOW - RETENTION_IDLE_MS), + provisioningCutoff: new Date(NOW - STRANDED_PROVISIONING_MS), + limit: STRANDED_BATCH, + }; + await findStranded({ ...args, names: [] }); + expect(queries).toBe(1); + await findStranded({ ...args, names: ["ws-a"] }); + expect(queries).toBe(3); + }); }); diff --git a/packages/trpc/src/router/cloud-workspace/retention.ts b/packages/trpc/src/router/cloud-workspace/retention.ts index 72b63542846..080df23a957 100644 --- a/packages/trpc/src/router/cloud-workspace/retention.ts +++ b/packages/trpc/src/router/cloud-workspace/retention.ts @@ -42,6 +42,10 @@ export const ORPHAN_MIN_AGE_MS = 7 * DAY_MS; */ export const REAP_BATCH = 200; export const ORPHAN_BATCH = 200; +/** Longer than any provision takes, its QStash retries included: minutes. */ +export const STRANDED_PROVISIONING_MS = DAY_MS; +/** Failed and stranded rows are rare: 2 failed and none stranded on 2026-11-19. */ +export const STRANDED_BATCH = 50; /** A backstop for a lost archive delete, which deletes inline: few a run. */ export const ARCHIVED_BATCH = 20; /** The archive's own delete runs at the grace; a day later it has had its retries. */ @@ -140,6 +144,125 @@ async function boxIsRunning(providerSandboxId: string): Promise { } } +export interface StrandedWorkspace { + id: string; + organizationId: string; + providerSandboxId: string; + status: "failed" | "provisioning"; +} + +/** Failed before `cutoff`; rows failed before `failed_at` existed count from their last write. */ +export function failedBefore(cutoff: Date): SQL { + return and( + eq(cloudWorkspaces.provider, "vercel"), + isNull(cloudWorkspaces.deletedAt), + eq(cloudWorkspaces.status, "failed"), + sql`coalesce(${cloudWorkspaces.failedAt}, ${cloudWorkspaces.updatedAt}) < ${cutoff.toISOString()}::timestamptz`, + ) as SQL; +} + +/** + * Provisioning since before `cutoff` with no boot since: every delivery of + * its job died. An unarchive moves a row to provisioning, so `updated_at` + * counts, not only `created_at`. + */ +export function strandedSince(cutoff: Date): SQL { + return and( + eq(cloudWorkspaces.provider, "vercel"), + isNull(cloudWorkspaces.deletedAt), + eq(cloudWorkspaces.status, "provisioning"), + lt(cloudWorkspaces.createdAt, cutoff), + lt(cloudWorkspaces.updatedAt, cutoff), + sql`not exists (select 1 from ${cloudWorkspaceBoots} where ${cloudWorkspaceBoots.cloudWorkspaceId} = ${cloudWorkspaces.id} and ${cloudWorkspaceBoots.startedAt} >= ${cutoff.toISOString()}::timestamptz)`, + ) as SQL; +} + +const strandedColumns = { + id: cloudWorkspaces.id, + organizationId: cloudWorkspaces.organizationId, + providerSandboxId: cloudWorkspaces.providerSandboxId, + status: cloudWorkspaces.status, +}; + +/** + * Stranded provisioning rows by their status alone: retiring fails them, so + * they leave. Failed rows only while their box is in `names`: a retired name + * is in no listing, so they leave too, and none can fill the batch forever. + */ +export async function findStranded({ + failedCutoff, + provisioningCutoff, + names, + limit, +}: { + failedCutoff: Date; + provisioningCutoff: Date; + names: string[]; + limit: number; +}): Promise { + const found = (await db + .select(strandedColumns) + .from(cloudWorkspaces) + .where(strandedSince(provisioningCutoff)) + .orderBy(asc(cloudWorkspaces.updatedAt)) + .limit(limit)) as StrandedWorkspace[]; + for ( + let i = 0; + i < names.length && found.length < limit; + i += NAME_LOOKUP_CHUNK + ) { + found.push( + ...((await db + .select(strandedColumns) + .from(cloudWorkspaces) + .where( + and( + failedBefore(failedCutoff), + inArray( + cloudWorkspaces.providerSandboxId, + names.slice(i, i + NAME_LOOKUP_CHUNK), + ), + ), + ) + .limit(limit - found.length)) as StrandedWorkspace[]), + ); + } + return found; +} + +/** + * Takes the box name off a failed or stranded row and queues the box's + * teardown, on the status and age read. A stranded row then fails, so the + * client stops waiting for it. False when the row changed since it was read. + */ +export async function retireStrandedWorkspace( + row: StrandedWorkspace, + cutoffs: { failed: Date; provisioning: Date }, +): Promise { + const stillStranded = + row.status === "failed" + ? failedBefore(cutoffs.failed) + : strandedSince(cutoffs.provisioning); + const retired = await retireAndTearDown( + { id: row.id, providerSandboxId: row.providerSandboxId, driveName: null }, + stillStranded, + ); + if (!retired) return false; + if (row.status === "provisioning") { + await failCloudWorkspace({ + scope: { cloudWorkspaceId: row.id, organizationId: row.organizationId }, + from: ["provisioning"], + set: { sandboxUrl: null }, + reason: "create_failed", + error: new Error( + `provisioning since before ${cutoffs.provisioning.toISOString()}`, + ), + expected: true, + }); + } + return true; +} + function archivedPastGrace(cutoff: Date): SQL { return and( eq(cloudWorkspaces.provider, "vercel"), @@ -222,6 +345,8 @@ export interface RetentionDeps { findArchivedPastGraceAmong: typeof findArchivedPastGraceAmong; reapArchived: typeof reapArchivedCloudWorkspace; reapUndatedArchived: typeof reapUndatedArchivedCloudWorkspace; + findStranded: typeof findStranded; + retireStrandedWorkspace: typeof retireStrandedWorkspace; tearDownOrphan: (providerSandboxId: string) => Promise; pause: (ms: number) => Promise; } @@ -236,6 +361,8 @@ const defaultDeps: RetentionDeps = { findArchivedPastGraceAmong, reapArchived: reapArchivedCloudWorkspace, reapUndatedArchived: reapUndatedArchivedCloudWorkspace, + findStranded, + retireStrandedWorkspace, tearDownOrphan: (providerSandboxId) => deleteSandboxAndDriveLater({ providerSandboxId, driveName: null }), pause: (ms) => new Promise((resolve) => setTimeout(resolve, ms)), @@ -260,6 +387,15 @@ export interface RetentionRun { failed: number; } | { skipped: "not production" }; + stranded: + | { + found: number; + retired: number; + skippedChanged: number; + deferred: number; + failed: number; + } + | { skipped: "not production" }; orphans: | { listed: number; @@ -276,8 +412,9 @@ export interface RetentionRun { * and unused for `RETENTION_IDLE_MS` fails as `snapshot_expired`, exactly as a * wake that met Vercel's expiry did, and its box is torn down. Then, in * production only: an archived box still there a day after its grace is - * deleted (its own delete was lost), and boxes we tagged that no row names - * are torn down. Every deployment shares the Vercel project, and a dev + * deleted (its own delete was lost); a row failed for `RETENTION_IDLE_MS`, or + * stuck provisioning for `STRANDED_PROVISIONING_MS`, gives up its box; and + * boxes we tagged that no row names are torn down. Every deployment shares the Vercel project, and a dev * database copied from production names production's boxes. */ export async function reapCloudWorkspaceRetention({ @@ -300,15 +437,19 @@ export async function reapCloudWorkspaceRetention({ const archived = production ? await reapArchived({ now, deps, boxes, outOfTime }) : notProduction; + const stranded = production + ? await reapStranded({ now, deps, boxes, retiredNames, outOfTime }) + : notProduction; const orphans = production ? await reapOrphans({ now, deps, boxes, retiredNames, outOfTime }) : notProduction; - const run = { idle, archived, orphans }; + const run = { idle, archived, stranded, orphans }; console.log("[cloud-workspace] retention run", JSON.stringify(run)); Sentry.logger.info("retention.run", flatten(run)); if ( idle.failed || ("failed" in archived && archived.failed) || + ("failed" in stranded && stranded.failed) || ("failed" in orphans && orphans.failed) ) { Sentry.captureMessage("cloud workspace retention run had failures", { @@ -421,6 +562,59 @@ async function reapArchived({ return counts; } +async function reapStranded({ + now, + deps, + boxes, + retiredNames, + outOfTime, +}: { + now: number; + deps: RetentionDeps; + boxes: ListedSandbox[]; + retiredNames: Set; + outOfTime: () => boolean; +}): Promise> { + const cutoffs = { + failed: new Date(now - RETENTION_IDLE_MS), + provisioning: new Date(now - STRANDED_PROVISIONING_MS), + }; + const rows = await deps.findStranded({ + failedCutoff: cutoffs.failed, + provisioningCutoff: cutoffs.provisioning, + names: boxes.map((box) => box.name), + limit: STRANDED_BATCH, + }); + const counts = { + found: rows.length, + retired: 0, + skippedChanged: 0, + deferred: 0, + failed: 0, + }; + for (const [index, row] of rows.entries()) { + if (outOfTime()) { + counts.deferred = rows.length - index; + break; + } + try { + if (!(await deps.retireStrandedWorkspace(row, cutoffs))) { + counts.skippedChanged++; + continue; + } + counts.retired++; + retiredNames.add(row.providerSandboxId); + await deps.pause(TEARDOWN_SPACING_MS); + } catch (error) { + counts.failed++; + Sentry.captureException(error, { + tags: { "cloud_workspace.id": row.id }, + }); + } + } + return counts; +} + function isOldAndSettled(box: ListedSandbox, now: number): boolean { return ( SETTLED_STATUSES.has(box.status) && @@ -488,6 +682,12 @@ function flatten(run: RetentionRun): Record { value, ]), ), + ...Object.fromEntries( + Object.entries(run.stranded).map(([key, value]) => [ + `stranded.${key}`, + value, + ]), + ), ...Object.fromEntries( Object.entries(run.orphans).map(([key, value]) => [ `orphans.${key}`, diff --git a/plans/done/20261118-1500-cloud-workspace-retention-reaper.md b/plans/done/20261118-1500-cloud-workspace-retention-reaper.md index 15661df5b80..f3a2e7cb8fb 100644 --- a/plans/done/20261118-1500-cloud-workspace-retention-reaper.md +++ b/plans/done/20261118-1500-cloud-workspace-retention-reaper.md @@ -87,8 +87,13 @@ None blocking. The QStash schedule must be created in the Upstash console by som - Decision: the idle stage retires names through `retireAndTearDown(row, condition)` with #9847's exact name, signature and module, and every box (with or without a drive) goes through the retried teardown job (orchestrator). Rationale: #9847's file then replaces this one with no change to `retention.ts`. The cost: the name and the status are now two writes, not one. The name goes first, so the window is an asleep row that names a box that does not exist, which the 410 path already handles. The other order would leave a failed row naming a box being deleted. `deleteSandboxAndDriveLater` on #9847 deletes a box without a drive inline, with no retry and no re-read; this PR queues it, because the orphan stage depends on the re-read just before delete. Date/Author: 2026-11-18, orchestrator. +- Decision: a stranded stage (production only, after the archived stage, batch 50) for the drive-liveness property that #9914's model found failing. It selects rows `provisioning` for `STRANDED_PROVISIONING_MS` (1 day, by `created_at`, `updated_at` and no boot since), and rows `failed` for `RETENTION_IDLE_MS` (by `coalesce(failed_at, updated_at)`) whose box is in this run's listing. It retires each through `retireAndTearDown` with the same status-and-age predicate as the condition; a stranded provisioning row then fails as `create_failed`. On main this covers boxes only; the drive half needs #9847's `drive_name` (see the rebase list in the PR). + Rationale: a stranded provisioning row leaves the selection by its status, a failed one because its retired name is in no listing, so no row can fill an oldest-first batch forever. Failed rows wait 30 days, because a failed row keeps its box for diagnosis. A call that does not complete is retried by a later run: a stranded row whose fail threw is selected again a day later (the rename moved `updated_at`); a box whose teardown was not queued is an orphan. On 2026-11-19 production had 2 failed rows (1 failed over 30 days ago) and no stranded ones. Date/Author: 2026-11-19, orchestrator. + ## Outcomes & Retrospective All seven parts of the brief are done except the drives. Workspace drives do not exist on main, so the drive half is a seam: `TeardownSandboxInput.driveName` is typed `null`, and #9847's version of `teardownSandbox` replaces this one at the same route with the same payload. The reaper's measure of last activity is deliberately later than Vercel's: the 412 rows whose `sandbox_expires_at` was moved forward on 2026-11-16 are reaped no earlier than 2026-12-16, and until then Vercel's own expiry and the 410 path still apply to them. Revision note (2026-11-18): aligned with #9847's `retireAndTearDown` and `deleteSandboxAndDriveLater`; the idle stage's single update became two writes, and teardowns are paced in the loop instead of by `delaySeconds`. + +Revision note (2026-11-19): added the stranded stage for failed and stuck-provisioning rows, for the drive-liveness property in #9914's model. From 4ee808aeeffb18052fcb999fbac4586fca9e0177 Mon Sep 17 00:00:00 2001 From: Satya Patel <14907857+saddlepaddle@users.noreply.github.com> Date: Thu, 19 Nov 2026 06:13:00 +0000 Subject: [PATCH 6/8] feat(cloud-workspace): the cleanup job applies the end-state lifecycle rules One rules table, production only, failing closed on an incomplete Vercel listing. Rule 1 retires workspaces past their wait (asleep 30 d, archived 7 d, failed 30 d, stuck creating 1 d) with one conditional write, then the retried teardown. Rule 2 deletes archived environments' goldens and, through goldensToDelete (as #10000), goldens past the newest 3. Rule 3 deletes boxes, drives and goldens no production row names after 7 days, and agents drives that are not current and no box mounts (#9943's listing checks). deleteSandbox and deleteDrive delete only when VERCEL_ENV is production. Archive, a failed create, the provision race and environment archive only set a status; archive no longer queues a delayed delete, and an unarchive after 7 days is refused. The release no longer prunes agents drives. --- docs/cloud-sandbox-mismatches.md | 60 +- packages/sandbox/src/agent-tools.test.ts | 56 - packages/sandbox/src/agent-tools.ts | 52 - packages/sandbox/src/environments/release.ts | 38 +- packages/trpc/src/lib/sandbox/index.ts | 8 +- packages/trpc/src/lib/sandbox/vercel.test.ts | 76 ++ packages/trpc/src/lib/sandbox/vercel.ts | 142 ++- .../cloud-workspace/cloud-workspace.test.ts | 28 +- .../router/cloud-workspace/cloud-workspace.ts | 46 +- .../router/cloud-workspace/golden-rotation.ts | 16 + .../router/cloud-workspace/provision.test.ts | 19 +- .../src/router/cloud-workspace/provision.ts | 30 +- .../src/router/cloud-workspace/reap.test.ts | 106 +- .../trpc/src/router/cloud-workspace/reap.ts | 70 +- .../router/cloud-workspace/retention.test.ts | 664 ++++++----- .../src/router/cloud-workspace/retention.ts | 1044 +++++++++-------- .../router/cloud-workspace/teardown-guard.ts | 21 + .../router/cloud-workspace/teardown.test.ts | 9 +- .../src/router/cloud-workspace/teardown.ts | 8 +- .../router/environment/environment.test.ts | 24 +- .../src/router/environment/environment.ts | 14 +- ...8-1500-cloud-workspace-retention-reaper.md | 5 + 22 files changed, 1325 insertions(+), 1211 deletions(-) create mode 100644 packages/trpc/src/router/cloud-workspace/golden-rotation.ts diff --git a/docs/cloud-sandbox-mismatches.md b/docs/cloud-sandbox-mismatches.md index 9b12879a997..21bd2b68b1e 100644 --- a/docs/cloud-sandbox-mismatches.md +++ b/docs/cloud-sandbox-mismatches.md @@ -785,38 +785,36 @@ never opens. **Vercel's snapshot expiry was silent, so we own retention now.** The app assumed a stopped box keeps its disk until someone deletes it. In fact Vercel -expired a workspace box's snapshot 30 days after its last use. Nothing told us: -the row learned only from the 410 that its next wake, agent turn or automation -dispatch got. A workspace nobody opened again stayed `asleep` over a box that -could not resume, and the box itself stayed in Vercel. New workspace boxes are -now created with `snapshotExpiration: 0`. The `reap-retention` job, every 6 hours, -(`router/cloud-workspace/retention.ts`) does three things. It moves a workspace -that is asleep and unused for 30 days to `failed`/`snapshot_expired`, the same -state as the 410. First `retireAndTearDown` gives the row a new box name and -queues `teardownSandbox`, which deletes the box only if no row names it; then -the row fails. A wake between the two finds no box and fails the row itself. "Unused" -means no boot started or stopped since then, with `sandbox_expires_at` and -`created_at` older too, so a wake in flight (its boot row is written before -Vercel resumes) keeps the box. Archive deletes its box at the end of the -7-day grace through one delayed QStash message, and Vercel's 30-day expiry used -to clean up after a lost one. That backstop is gone, so in production the job -also deletes an archived box still listed a day after its grace, through the -archive's own delete stage (`reapUndatedArchivedCloudWorkspace` for the rows -archived before `deleted_at` existed). It matches rows against one listing of -every box, so a box already gone costs no provider call. It also takes the box -from a row failed 30 days ago whose box is still listed, and from a row stuck -in `provisioning` for a day (every delivery of its job died), which then fails -as `create_failed`. A retired name is in no listing and a failed row is no -longer provisioning, so these rows leave the selection. In production only, -the same job also tears down -tagged boxes that no row and no unarchived environment names, once they are -7 days old: every deployment shares one Vercel project, so a dev or preview -database would see production's boxes as orphans. Boxes created before this +expired a workspace box's snapshot 30 days after its last use, and nothing told +us: the row learned only from the 410 its next wake got. New workspace boxes +are now created with `snapshotExpiration: 0`, and one cleanup job +(`router/cloud-workspace/retention.ts`, every 6 hours, production only) is the +only thing that deletes for age. It applies three rules from one table: + +1. A workspace not in use is retired after its wait: asleep 30 days after last + use, archived 7 days, failed 30 days, stuck creating with no boot 1 day. + Retiring is one conditional write (still in that status, still past the + wait) that takes the box name off the row, then the retried teardown job + deletes the box. A wake that lands first wins. An unarchive after the 7 days + is refused even before the job runs. +2. An environment keeps its active golden and its newest others, 3 in all; an + archived environment keeps none. On main an environment records only its + active golden, so the job deletes archived environments' goldens. +3. A box, drive or golden no production row names goes 7 days after it was + made; an agents drive goes once it is not the current release's and no box, + stopped ones included, mounts it. + +Everything else only sets a status: archive (its 60 s stop stays), a failed +create and a 410 on wake. Only production deletes: `deleteSandbox` and +`deleteDrive` log "would delete" unless `VERCEL_ENV` is `production`, because +every deployment shares one Vercel project and a preview database copied from +production names production's boxes. Staging deploys as production on the +production database, so it counts. The job reads the whole box and drive +listing first and deletes nothing if it cannot. Boxes created before this change keep a 30-day expiry until `bun run backfill-snapshot-expiration ---apply` (`packages/sandbox`) is run. That run changes only snapshots taken -afterwards, so until a box's next stop the 410 path stays its fallback. -**Open:** drives are not reaped yet. No row names a workspace drive on main, -so that half waits for the drive columns (#9847). +--apply` (`packages/sandbox`) is run by hand. **Open:** no row names a +workspace drive on main; #9847's `drive_name` and `golden_drive_name` must +join rule 3's names when it lands. **Any call on a `Sandbox` object resumes a stopped session.** The SDK wraps `runCommand`, `writeFiles`, `extendTimeout` and the rest in `withResume`: on a diff --git a/packages/sandbox/src/agent-tools.test.ts b/packages/sandbox/src/agent-tools.test.ts index cb8744da80d..09aae7263fd 100644 --- a/packages/sandbox/src/agent-tools.test.ts +++ b/packages/sandbox/src/agent-tools.test.ts @@ -10,18 +10,15 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { type AgentToolsManifest, - agentToolsDrivesToPrune, agentToolsGaps, agentToolsManifest, type DriveHost, describeAgentToolsPublish, parseAgentToolsManifest, parseBuilderResults, - pruneAgentToolsDrives, publishAgentToolsDrive, publishAgentToolsDrives, type RegionPublish, - referencedAgentToolsDrives, renderBuilderScript, } from "./agent-tools"; import { type Assets, loadAssets, sha256 } from "./manifest"; @@ -387,59 +384,6 @@ describe("agentToolsGaps", () => { }); }); -test("referenced drives: every row's bundle in its region, and the recent bundles in every region", () => { - const older = "1".repeat(64); - expect( - [ - ...referencedAgentToolsDrives({ - rows: [ - { region: "iad1", bundleSha: older }, - { region: "fra1", bundleSha: null }, - ], - recentBundles: [BUNDLE], - regions: ["sfo1", "iad1"], - }), - ].sort(), - ).toEqual([ - "agent-tools-iad1-0123456789ab", - "agent-tools-iad1-111111111111", - "agent-tools-sfo1-0123456789ab", - ]); -}); - -describe("pruning", () => { - test("deletes only agent-tools drives no environment references", () => { - expect( - agentToolsDrivesToPrune( - [ - "agent-tools-sfo1-0123456789ab", - "agent-tools-sfo1-aaaaaaaaaaaa", - "agent-tools-iad1-0123456789ab", - "agent-tools-sfo1", - "agent-tools-builder-sfo1-xyz", - ], - new Set(["agent-tools-sfo1-0123456789ab"]), - ), - ).toEqual([ - "agent-tools-sfo1-aaaaaaaaaaaa", - "agent-tools-iad1-0123456789ab", - ]); - }); - - test("keeps a drive a sandbox holds for the next release", async () => { - const { host } = fakeHost({ - "agent-tools-sfo1-aaaaaaaaaaaa": {}, - "agent-tools-sfo1-bbbbbbbbbbbb": { attachedTo: "box" }, - }); - expect( - await pruneAgentToolsDrives({ host, referenced: new Set() }), - ).toEqual({ - deleted: ["agent-tools-sfo1-aaaaaaaaaaaa"], - kept: ["agent-tools-sfo1-bbbbbbbbbbbb"], - }); - }); -}); - async function runScript(script: string) { const proc = Bun.spawn(["bash", "-c", script], { stdout: "pipe", diff --git a/packages/sandbox/src/agent-tools.ts b/packages/sandbox/src/agent-tools.ts index f289e3143a0..6d7a506208a 100644 --- a/packages/sandbox/src/agent-tools.ts +++ b/packages/sandbox/src/agent-tools.ts @@ -452,58 +452,6 @@ export function agentToolsGaps( }); } -/** - * Every drive a box may still mount: each environment row's bundle in its - * region, and the newest released bundles in every region, so a rollback and - * an environment created mid-release still find theirs. - */ -export function referencedAgentToolsDrives(args: { - rows: readonly { region: string; bundleSha: string | null }[]; - recentBundles: readonly string[]; - regions: readonly string[]; -}): Set { - return new Set([ - ...args.rows.flatMap((row) => - row.bundleSha ? [sandboxAgentToolsDrive(row.region, row.bundleSha)] : [], - ), - ...args.recentBundles.flatMap((sha) => - args.regions.map((region) => sandboxAgentToolsDrive(region, sha)), - ), - ]); -} - -/** Agent-tools drives no environment row's bundle and region name any more. */ -export function agentToolsDrivesToPrune( - drives: readonly string[], - referenced: ReadonlySet, -): string[] { - return drives.filter( - (name) => - SANDBOX_AGENT_TOOLS_DRIVE_PATTERN.test(name) && !referenced.has(name), - ); -} - -/** Returns the drives it could not delete; a drive a sandbox holds is retried next release. */ -export async function pruneAgentToolsDrives(args: { - host: DriveHost; - referenced: ReadonlySet; -}): Promise<{ deleted: string[]; kept: string[] }> { - const deleted: string[] = []; - const kept: string[] = []; - for (const name of agentToolsDrivesToPrune( - (await args.host.list("agent-tools-")).map((drive) => drive.name), - args.referenced, - )) { - try { - await args.host.remove(name); - deleted.push(name); - } catch { - kept.push(name); - } - } - return { deleted, kept }; -} - interface VercelCredentials { token: string; teamId: string; diff --git a/packages/sandbox/src/environments/release.ts b/packages/sandbox/src/environments/release.ts index 95dd3faf6c6..3f388d4ff27 100644 --- a/packages/sandbox/src/environments/release.ts +++ b/packages/sandbox/src/environments/release.ts @@ -59,8 +59,6 @@ process.env.SKIP_ENV_VALIDATION ??= "1"; const SKIP_IMAGE = process.argv.includes("--skip-image"); const KEEP_OLD = process.argv.includes("--keep-old"); const PRODUCTION = process.argv.includes("--production"); -/** Released bundles whose drives stay in every region, so a rollback to one of them has its drives. */ -const KEPT_RELEASED_BUNDLES = 3; const FOLLOWED_UPSTREAM = ["claude", "codex", "chrome", "opencode", "pi"]; /** The environment the release rebuilds; without it, the release creates one named INTERNAL_NAME. */ const ENVIRONMENT_ID = process.env.SUPERSET_INTERNAL_ENVIRONMENT_ID; @@ -521,7 +519,7 @@ const { db, dbWs } = await import("@superset/db/client"); const { environments, environmentRepositories } = await import( "@superset/db/schema" ); -const { desc, eq } = await import("drizzle-orm"); +const { eq } = await import("drizzle-orm"); const { releaseRepositoryWhere } = await import( "./utils/releaseRepositoryWhere" ); @@ -605,39 +603,7 @@ if (!previous) `rows: created; set SUPERSET_INTERNAL_ENVIRONMENT_ID=${internal.id} so the next release updates it`, ); -// One Vercel project holds every database's drives (staging shares -// production's rows; preview branches copy them), so only the production -// release, whose rows the others mirror, may delete one. -if (!PRODUCTION) { - log("agent-tools: prune skipped; only --production prunes drives"); -} else { - try { - const { sandboxBundles } = await import("@superset/db/schema"); - const rows = await db.query.environments.findMany({ - columns: { region: true, bundleSha: true }, - }); - const recent = await db - .select({ sha: sandboxBundles.sha }) - .from(sandboxBundles) - .orderBy(desc(sandboxBundles.releasedAt)) - .limit(KEPT_RELEASED_BUNDLES); - const { deleted, kept } = await agentTools.pruneAgentToolsDrives({ - host: driveHost, - referenced: agentTools.referencedAgentToolsDrives({ - rows, - recentBundles: [bundle.sha256, ...recent.map((row) => row.sha)], - regions: SANDBOX_REGION_IDS, - }), - }); - log( - `agent-tools: pruned ${deleted.length}${kept.length ? `; ${kept.join(", ")} still attached, next release retries` : ""}`, - ); - } catch (error) { - console.log( - `::warning::agent-tools: prune failed: ${String(error).slice(0, 300)}`, - ); - } -} +// Old agents drives are deleted by the API's cleanup job (rule 3), once no box mounts them. if ( previous && diff --git a/packages/trpc/src/lib/sandbox/index.ts b/packages/trpc/src/lib/sandbox/index.ts index b3a7985f15a..993e5f82437 100644 --- a/packages/trpc/src/lib/sandbox/index.ts +++ b/packages/trpc/src/lib/sandbox/index.ts @@ -55,14 +55,20 @@ export { export { type ActiveWorkspaceSandbox, type BoxSession, + canDelete, + deleteDrive, deleteSandbox, describeSandbox, findSandboxSession, HOST_SERVICE_PORT, + type ListedDrive, type ListedSandbox, + type ListingPage, listActiveWorkspaceSandboxes, - listAllSandboxes, + listProjectDrives, + listProjectSandboxes, listSandboxSessions, + listWhole, MEMORY_MB_PER_VCPU, promoteSandboxToEnvironment, provisionSandbox, diff --git a/packages/trpc/src/lib/sandbox/vercel.test.ts b/packages/trpc/src/lib/sandbox/vercel.test.ts index e7fe57e7ab4..f455966eb3c 100644 --- a/packages/trpc/src/lib/sandbox/vercel.test.ts +++ b/packages/trpc/src/lib/sandbox/vercel.test.ts @@ -11,11 +11,14 @@ import { stub } from "../../../test/stub"; import { agentToolsMounts, applySandboxPolicy, + canDelete, + deleteDrive, deleteSandbox, describeSandbox, findSandboxSession, listActiveWorkspaceSandboxes, listSandboxSessions, + listWhole, promoteSandboxToEnvironment, provisionSandbox, type SandboxClaim, @@ -308,6 +311,79 @@ describe("a caller that is not a wake leaves a stopped box stopped", () => { }); }); +describe("only production deletes", () => { + stub(Sandbox, { + get: async () => { + throw new Error("reached Vercel"); + }, + }); + stub(Drive, { + get: async () => { + throw new Error("reached Vercel"); + }, + }); + + test("outside production a box or drive delete logs and returns without reaching Vercel", async () => { + const vercelEnv = process.env.VERCEL_ENV; + process.env.VERCEL_ENV = "preview"; + try { + await deleteSandbox("ws-box"); + await deleteDrive("drive-1"); + } finally { + process.env.VERCEL_ENV = vercelEnv; + } + expect(canDelete("production")).toBe(true); + expect(canDelete("preview")).toBe(false); + expect(canDelete(undefined)).toBe(false); + }); +}); + +describe("listWhole", () => { + const pages = (...list: Array<{ items: string[]; next: string | null }>) => { + let index = 0; + return async () => { + const page = list[index++]; + if (!page) throw new Error("no such page"); + return { items: page.items.map((name) => ({ name })), next: page.next }; + }; + }; + + test("returns every page, and throws on a repeated cursor, a missing known name or an empty listing", async () => { + expect( + await listWhole({ + fetchPage: pages( + { items: ["a"], next: "c1" }, + { items: ["b"], next: null }, + ), + expected: ["b"], + nonEmpty: true, + }), + ).toEqual([{ name: "a" }, { name: "b" }]); + await expect( + listWhole({ + fetchPage: pages( + { items: ["a"], next: "c1" }, + { items: ["b"], next: "c1" }, + ), + nonEmpty: true, + }), + ).rejects.toThrow("repeated cursor"); + await expect( + listWhole({ + fetchPage: pages({ items: ["a"], next: null }), + expected: ["ws-running"], + nonEmpty: true, + }), + ).rejects.toThrow("lacks ws-running"); + await expect( + listWhole({ + fetchPage: pages({ items: [], next: null }), + nonEmpty: true, + }), + ).rejects.toThrow("empty"); + }); +}); + describe("provisionSandbox", () => { let created: { snapshotExpiration?: number } | undefined; stub(Sandbox, { diff --git a/packages/trpc/src/lib/sandbox/vercel.ts b/packages/trpc/src/lib/sandbox/vercel.ts index ba74cbbb802..f062aa5fb05 100644 --- a/packages/trpc/src/lib/sandbox/vercel.ts +++ b/packages/trpc/src/lib/sandbox/vercel.ts @@ -992,27 +992,143 @@ export interface ListedSandbox { /** Our `kind` tag; absent on a box made before tags or not made by us. */ kind: string | undefined; createdAt: number; - statusUpdatedAt: number; status: string; + /** What it mounts, running or stopped: a stopped box mounts it again on its next boot. */ + mounts: string[]; } -/** Every box in the project, whatever its age or tags. */ -export async function listAllSandboxes(): Promise { - const found: ListedSandbox[] = []; - for await (const sandbox of await Sandbox.list(credentials())) { - found.push({ - name: sandbox.name, - kind: sandbox.tags?.kind, - createdAt: sandbox.createdAt, - statusUpdatedAt: sandbox.statusUpdatedAt ?? sandbox.updatedAt, - status: sandbox.status, - }); +export interface ListedDrive { + name: string; + createdAt: number; + attachedTo: string | undefined; +} + +export interface ListingPage { + items: T[]; + next: string | null; +} + +export const MAX_LISTING_PAGES = 10_000; + +/** + * Every page of a listing, or a throw: each page answered, no cursor seen + * twice, and the names the caller knows exist all in it. A deleter that acts + * on a partial listing deletes what it did not see named. + */ +export async function listWhole({ + fetchPage, + expected = [], + nonEmpty, +}: { + fetchPage: (cursor: string | null) => Promise>; + expected?: readonly string[]; + nonEmpty: boolean; +}): Promise { + const items: T[] = []; + const seen = new Set(); + let cursor: string | null = null; + for (let pages = 0; ; pages++) { + if (pages >= MAX_LISTING_PAGES) { + throw new Error(`listing has more than ${pages} pages`); + } + const page = await fetchPage(cursor); + items.push(...page.items); + if (page.next === null) break; + if (seen.has(page.next)) { + throw new Error(`listing repeated cursor ${page.next.slice(0, 16)}`); + } + seen.add(page.next); + cursor = page.next; } - return found; + if (nonEmpty && items.length === 0) throw new Error("listing is empty"); + const names = new Set(items.map((item) => item.name)); + const missing = expected.filter((name) => !names.has(name)); + if (missing.length) throw new Error(`listing lacks ${missing.join(", ")}`); + return items; +} + +/** The page size #9943 used against the same endpoint. */ +const LISTING_PAGE_SIZE = 50; + +/** Every box in the project, whatever its status, tags or database; throws unless whole. */ +export function listProjectSandboxes( + expected: readonly string[], +): Promise { + return listWhole({ + expected, + nonEmpty: true, + fetchPage: async (cursor) => { + const page = await Sandbox.list({ + ...credentials(), + limit: LISTING_PAGE_SIZE, + ...(cursor ? { cursor } : {}), + }); + return { + items: page.sandboxes.map((sandbox) => ({ + name: sandbox.name, + kind: sandbox.tags?.kind, + createdAt: sandbox.createdAt, + status: sandbox.status, + mounts: Object.values(sandbox.mounts ?? {}).map( + (mount) => mount.drive, + ), + })), + next: page.pagination.next, + }; + }, + }); +} + +/** Every drive in the project; throws unless whole. */ +export function listProjectDrives(): Promise { + return listWhole({ + nonEmpty: false, + fetchPage: async (cursor) => { + const page = await Drive.list({ + ...credentials(), + limit: LISTING_PAGE_SIZE, + ...(cursor ? { cursor } : {}), + }); + return { + items: page.drives.map((drive) => ({ + name: drive.name, + createdAt: drive.createdAt.getTime(), + attachedTo: drive.currentSandboxName, + })), + next: page.pagination.next, + }; + }, + }); +} + +/** + * Only production deletes. Staging deploys as production on the same + * database, so it counts; a preview or local database, copied from + * production, names production's boxes and only logs. + */ +export function canDelete(vercelEnv = process.env.VERCEL_ENV): boolean { + return vercelEnv === "production"; +} + +function wouldDelete(kind: string, name: string): boolean { + if (canDelete()) return false; + console.log(`[sandbox] would delete ${kind} ${name}: not production`); + return true; +} + +/** Throws while a box holds it; a drive already gone is the state we wanted. */ +export async function deleteDrive(name: string): Promise { + if (wouldDelete("drive", name)) return; + const drive = await Drive.get({ ...credentials(), name }).catch((error) => { + if (isNotFound(error)) return null; + throw error; + }); + await drive?.delete(); } /** Best-effort: a sandbox already gone is the state we wanted. */ export async function deleteSandbox(providerSandboxId: string): Promise { + if (wouldDelete("sandbox", providerSandboxId)) return; const sandbox = await getSandbox(providerSandboxId); if (!sandbox) return; // Snapshots outlive a sandbox by default and keep billing storage. diff --git a/packages/trpc/src/router/cloud-workspace/cloud-workspace.test.ts b/packages/trpc/src/router/cloud-workspace/cloud-workspace.test.ts index 09f35d9d042..2115f531801 100644 --- a/packages/trpc/src/router/cloud-workspace/cloud-workspace.test.ts +++ b/packages/trpc/src/router/cloud-workspace/cloud-workspace.test.ts @@ -332,13 +332,33 @@ describe("an address read that finds the box stopped", () => { }); describe("cloudWorkspace.delete when the reap cannot be queued", () => { - test("ends every open boot as archived before the final settle", async () => { + test("archive only sets a status: it ends no boot and deletes nothing", async () => { + endedBoots = []; expect(await caller.cloudWorkspace.delete({ id: WORKSPACE_ID })).toEqual({ deleted: true, }); - expect(endedBoots).toEqual([ - [WORKSPACE_ID, { sessionId: null, reason: "archived" }], - ]); + expect(endedBoots).toEqual([]); + }); +}); + +describe("cloudWorkspace.unarchive", () => { + test("is refused once the 7 days are over, before the cleanup job has run", async () => { + const before = { ...ROW }; + Object.assign(ROW, { + status: "archived", + deletedAt: new Date(Date.now() - 8 * 24 * 60 * 60 * 1000), + updatedAt: new Date(), + }); + try { + expect( + await caller.cloudWorkspace.unarchive({ id: WORKSPACE_ID }), + ).toEqual({ unarchived: false }); + } finally { + Object.assign(ROW, before, { + deletedAt: undefined, + updatedAt: undefined, + }); + } }); }); diff --git a/packages/trpc/src/router/cloud-workspace/cloud-workspace.ts b/packages/trpc/src/router/cloud-workspace/cloud-workspace.ts index c9bd47c194d..647b2421a1f 100644 --- a/packages/trpc/src/router/cloud-workspace/cloud-workspace.ts +++ b/packages/trpc/src/router/cloud-workspace/cloud-workspace.ts @@ -37,7 +37,6 @@ import { logControl } from "../../lib/control-log"; import { callMoves } from "../../lib/moves"; import { nudge, readWorkspaceForwards } from "../../lib/realtime"; import { - deleteSandbox, describeSandbox, HOST_SERVICE_PORT, listRemoteBranches, @@ -69,21 +68,13 @@ import { import { recordCloudWorkspaceActivity } from "./activity"; import { withClientStatus } from "./client-status"; import { isLiveStatus } from "./live"; -import { - assertMayBoot, - queueSettleWorkspace, - settleWorkspaceNow, -} from "./meter"; +import { assertMayBoot, queueSettleWorkspace } from "./meter"; import { nextSandboxNameFor } from "./provision"; import { listPullRequests } from "./pull-requests"; -import { queueReap, reapWorkspaceFiles } from "./reap"; +import { archiveWaitIsOver, queueReap } from "./reap"; import { cloudWorkspaceRecordRouter } from "./record"; import { recordSandboxAwake } from "./sandbox-expiry"; -import { - recordBootEnd, - recordBoxStopped, - recordStopSeen, -} from "./sandbox-sleep"; +import { recordBoxStopped, recordStopSeen } from "./sandbox-sleep"; import { cloudWorkspaceSharingRouter } from "./sharing"; import { machineSizeNotAvailable, @@ -870,8 +861,9 @@ export const cloudWorkspaceRouter = { const archivedAt = new Date(); // From any state, provisioning included: the job checks the row - // before it marks it ready and tears its box down when this won. - // The box keeps running until the reap stops it, so an undo is instant. + // before it marks it ready and stops its box when this won. + // The box keeps running until the reap stops it, so an undo is instant; + // the cleanup job deletes it once the grace is over. const archived = await transitionCloudWorkspace({ id: row.id, from: ["provisioning", "awake", "asleep", "failed"], @@ -898,27 +890,12 @@ export const cloudWorkspaceRouter = { await queueReap( { cloudWorkspaceId: row.id, archivedAt: archivedAt.toISOString() }, row, - ).catch(async (error) => { + ).catch((error) => console.error( - `[cloud-workspace] could not queue the reap for ${row.id}`, + `[cloud-workspace] could not stop the box of ${row.id}; the cleanup job deletes it after the grace`, error, - ); - await recordBootEnd( - { cloudWorkspaceId: row.id, organizationId: row.organizationId }, - { - sessionId: null, - reason: "archived", - }, - ); - await settleWorkspaceNow(row.id).catch((meterError) => - console.error( - `[cloud-workspace] ${row.id} final meter failed`, - meterError, - ), - ); - await deleteSandbox(row.providerSandboxId); - await reapWorkspaceFiles(row.id); - }); + ), + ); } await recordCloudWorkspaceActivity( db, @@ -940,6 +917,9 @@ export const cloudWorkspaceRouter = { .input(z.object({ id: z.string().uuid() })) .mutation(async ({ ctx, input }) => { const row = await loadVisibleWorkspace(ctx, input.id); + if (row.status === "archived" && archiveWaitIsOver(row)) { + return { unarchived: false }; + } const resumable = row.status === "archived" && row.provider === "vercel" && diff --git a/packages/trpc/src/router/cloud-workspace/golden-rotation.ts b/packages/trpc/src/router/cloud-workspace/golden-rotation.ts new file mode 100644 index 00000000000..4b7f197a6a3 --- /dev/null +++ b/packages/trpc/src/router/cloud-workspace/golden-rotation.ts @@ -0,0 +1,16 @@ +/** Rule 2: an environment keeps its active golden and its newest others, this many in all. */ +export const KEPT_GOLDENS = 3; + +/** + * Goldens past the active one and the `KEPT_GOLDENS - 1` newest others. + * Takes undeleted goldens, newest first; never returns the active one. + * The same as `goldensToDelete` in #10000's environment-build policy. + */ +export function goldensToDelete( + goldens: readonly T[], + activeRef: string, +): T[] { + return goldens + .filter((golden) => golden.goldenRef !== activeRef) + .slice(KEPT_GOLDENS - 1); +} diff --git a/packages/trpc/src/router/cloud-workspace/provision.test.ts b/packages/trpc/src/router/cloud-workspace/provision.test.ts index 83d1dacd057..cfa68314784 100644 --- a/packages/trpc/src/router/cloud-workspace/provision.test.ts +++ b/packages/trpc/src/router/cloud-workspace/provision.test.ts @@ -10,6 +10,7 @@ import * as meter from "./meter"; import { provisionCloudWorkspace } from "./provision"; const logged: unknown[] = []; +const boxCalls: string[] = []; const captured: unknown[] = []; let claimError: unknown; stub(db.query.cloudWorkspaces, { @@ -24,8 +25,13 @@ stub(sandbox, { buildSandboxClaim: async () => { throw claimError; }, - deleteSandbox: async () => {}, - stopSandbox: async () => ({ sessionId: null }), + deleteSandbox: async (name: string) => { + boxCalls.push(`delete:${name}`); + }, + stopSandbox: async (name: string) => { + boxCalls.push(`stop:${name}`); + return { sessionId: null }; + }, }); stub(realtime, { nudge: () => {} }); stub(chatMoves, { failChatMovesTo: async () => {} }); @@ -56,6 +62,15 @@ function rowUpdate(moved: boolean, sets: unknown[] = []) { describe("a provision that Autumn refuses", () => { stub(db, rowUpdate(true)); + test("a failed create only stops its box and marks the row failed: the cleanup job deletes it later", async () => { + boxCalls.length = 0; + claimError = new Error("boom"); + expect(await provisionCloudWorkspace({ cloudWorkspaceId: "ws_1" })).toBe( + "failed", + ); + expect(boxCalls).toEqual(["stop:ws-box"]); + }); + test("fails the workspace with a warning and opens no Sentry issue", async () => { logged.length = 0; captured.length = 0; diff --git a/packages/trpc/src/router/cloud-workspace/provision.ts b/packages/trpc/src/router/cloud-workspace/provision.ts index 8bc3b4c365c..4716d4a6446 100644 --- a/packages/trpc/src/router/cloud-workspace/provision.ts +++ b/packages/trpc/src/router/cloud-workspace/provision.ts @@ -12,7 +12,6 @@ import { nudge } from "../../lib/realtime"; import { bootPhases, buildSandboxClaim, - deleteSandbox, provisionSandbox, SandboxNotReadyError, settleSandbox, @@ -161,10 +160,10 @@ async function provision( }, }); if (!ready) { - // Deleted while the box was being made: the delete won the row, so - // the box it never knew about goes with it. A duplicate delivery of - // this job loses the row to the first one and must leave the box be: - // the name is per workspace, so it is the same box. + // Archived while the box was being made: the archive won the row, so + // the box is stopped and left to the cleanup job's teardown. A + // duplicate delivery of this job loses the row to the first one and + // must leave the box be: the name is per workspace, so it is the same box. const current = await db.query.cloudWorkspaces.findFirst({ where: eq(cloudWorkspaces.id, row.id), columns: { status: true }, @@ -177,7 +176,7 @@ async function provision( error, ), ); - await deleteSandbox(providerSandboxId); + await stopSandbox(providerSandboxId); if (bootId) await releaseBoot(bootId, "provision_abandoned"); } await naming.catch(() => {}); @@ -238,18 +237,13 @@ async function provision( ), ); } - // A box that booted but whose host-service never answered is kept for - // diagnosis, stopped: its boot log, desktop and shell are a resume - // away, it costs storage rather than compute meanwhile, and a delete - // from the sidebar still removes it. Any other failure must not leak - // a box, since billing started at provision. - const keepForDiagnosis = error instanceof SandboxNotReadyError; - await (keepForDiagnosis ? stopSandbox : deleteSandbox)( - providerSandboxId, - ).catch((teardownError) => { + // A failed create only stops the box: it costs storage, not compute, + // its boot log is a resume away, and the cleanup job deletes it 30 + // days after the failure (rule 1). + await stopSandbox(providerSandboxId).catch((stopError) => { console.error( - `[cloud-workspace] ${keepForDiagnosis ? "could not stop" : "leaked"} sandbox ${providerSandboxId}`, - teardownError, + `[cloud-workspace] could not stop sandbox ${providerSandboxId}`, + stopError, ); }); await failCloudWorkspace({ @@ -259,7 +253,7 @@ async function provision( bootId, }, from: ["provisioning", "awake", "asleep"], - set: keepForDiagnosis ? { providerSandboxId } : {}, + set: { providerSandboxId }, reason: provisionFailureReason(error), error, }); diff --git a/packages/trpc/src/router/cloud-workspace/reap.test.ts b/packages/trpc/src/router/cloud-workspace/reap.test.ts index c1723e69bf0..f117c940422 100644 --- a/packages/trpc/src/router/cloud-workspace/reap.test.ts +++ b/packages/trpc/src/router/cloud-workspace/reap.test.ts @@ -101,18 +101,15 @@ stub(meter, { }, }); -const { - queueReap, - reapArchivedCloudWorkspace, - reapUndatedArchivedCloudWorkspace, -} = await import("./reap"); +const { archiveWaitIsOver, queueReap, reapArchivedCloudWorkspace } = + await import("./reap"); const archivedAt = new Date("2026-10-01T00:00:00.000Z"); const input = { cloudWorkspaceId: "00000000-0000-0000-0000-000000000001", archivedAt: archivedAt.toISOString(), }; -const SEVEN_DAYS = 7 * 24 * 60 * 60; +const DAY_MS = 24 * 60 * 60 * 1000; const BOX = { organizationId: "org_1", @@ -128,14 +125,9 @@ describe("queueReap", () => { recordedAsleep = []; }); - test("queues the delete after seven days and the stop after a minute", async () => { + test("queues only the stop, after a minute: the cleanup job deletes the box", async () => { await queueReap(input, BOX); expect(queued).toEqual([ - { - path: "/api/cloud-workspaces/reap", - body: { ...input, stage: "delete" }, - delaySeconds: SEVEN_DAYS, - }, { path: "/api/cloud-workspaces/reap", body: { ...input, stage: "stop" }, @@ -152,9 +144,7 @@ describe("queueReap", () => { expect(recordedAsleep).toEqual([ [input.cloudWorkspaceId, { sessionId: "session_1" }], ]); - expect(queued.map((job) => (job.body as { stage: Stage }).stage)).toEqual([ - "delete", - ]); + expect(queued).toEqual([]); }); test("a stop that could not be queued ends the stopped session's boot as archived", async () => { @@ -165,12 +155,6 @@ describe("queueReap", () => { [input.cloudWorkspaceId, { sessionId: "session_1", reason: "archived" }], ]); }); - - test("a delete that could not be queued throws so the caller deletes the box", async () => { - failingStage = "delete"; - await expect(queueReap(input, BOX)).rejects.toThrow("qstash down"); - expect(queued).toEqual([]); - }); }); describe("reapArchivedCloudWorkspace", () => { @@ -201,13 +185,6 @@ describe("reapArchivedCloudWorkspace", () => { ]); }); - test("the delete stage ends every boot still open as archived", async () => { - await reapArchivedCloudWorkspace({ ...input, stage: "delete" }); - expect(endedBoots).toEqual([ - [input.cloudWorkspaceId, { sessionId: null, reason: "archived" }], - ]); - }); - test("the stop stage runs the archive hook, stops the box and queues its final settle", async () => { expect(await reapArchivedCloudWorkspace({ ...input, stage: "stop" })).toBe( "stopped", @@ -257,25 +234,14 @@ describe("reapArchivedCloudWorkspace", () => { expect(meterCalls).toEqual([]); }); - test("the delete stage settles the box's usage before deleting it", async () => { - expect( - await reapArchivedCloudWorkspace({ ...input, stage: "delete" }), - ).toBe("reaped"); - expect(meterCalls).toEqual(["settle"]); - expect(sandboxCalls).toEqual(["delete:ws-box"]); - }); - - test("the delete stage removes the files sent to the workspace", async () => { - await reapArchivedCloudWorkspace({ ...input, stage: "delete" }); - expect(reapedFiles).toEqual([`file-of-${input.cloudWorkspaceId}`]); - }); - - test("a delete still deletes when the final settle fails: at most one lock window goes unbilled", async () => { - meterFailure = new Error("autumn down"); + test("the delete stage of an archive queued before the cleanup job deletes nothing", async () => { expect( await reapArchivedCloudWorkspace({ ...input, stage: "delete" }), - ).toBe("reaped"); - expect(sandboxCalls).toEqual(["delete:ws-box"]); + ).toBe("skipped"); + expect(sandboxCalls).toEqual([]); + expect(meterCalls).toEqual([]); + expect(endedBoots).toEqual([]); + expect(reapedFiles).toEqual([]); }); for (const stage of ["stop", "delete"] as const) { @@ -319,40 +285,26 @@ describe("reapArchivedCloudWorkspace", () => { }); }); -describe("reapUndatedArchivedCloudWorkspace", () => { - test("deletes the box of an archived row with no archive date only while it is still archived and unchanged", async () => { - const archivedBefore = new Date("2026-11-10T00:00:00.000Z"); - const undated = { - id: input.cloudWorkspaceId, - status: "archived", - deletedAt: null, - provider: "vercel", - providerSandboxId: "ws-undated", - updatedAt: new Date("2026-10-01T00:00:00.000Z"), - }; - sandboxCalls = []; - row = { ...undated, status: "asleep" }; +describe("archiveWaitIsOver", () => { + test("an unarchive is refused from day 7, counting an undated archive from its last write", () => { + const now = archivedAt.getTime(); expect( - await reapUndatedArchivedCloudWorkspace({ - cloudWorkspaceId: undated.id, - archivedBefore, - }), - ).toBe("skipped"); - row = { ...undated, updatedAt: new Date("2026-11-12T00:00:00.000Z") }; + archiveWaitIsOver( + { deletedAt: new Date(now - 6 * DAY_MS), updatedAt: new Date(0) }, + now, + ), + ).toBe(false); expect( - await reapUndatedArchivedCloudWorkspace({ - cloudWorkspaceId: undated.id, - archivedBefore, - }), - ).toBe("skipped"); - expect(sandboxCalls).toEqual([]); - row = undated; + archiveWaitIsOver( + { deletedAt: new Date(now - 7 * DAY_MS), updatedAt: new Date(now) }, + now, + ), + ).toBe(true); expect( - await reapUndatedArchivedCloudWorkspace({ - cloudWorkspaceId: undated.id, - archivedBefore, - }), - ).toBe("reaped"); - expect(sandboxCalls).toEqual(["delete:ws-undated"]); + archiveWaitIsOver( + { deletedAt: null, updatedAt: new Date(now - 8 * DAY_MS) }, + now, + ), + ).toBe(true); }); }); diff --git a/packages/trpc/src/router/cloud-workspace/reap.ts b/packages/trpc/src/router/cloud-workspace/reap.ts index a5db0fe33d2..5acbbba3791 100644 --- a/packages/trpc/src/router/cloud-workspace/reap.ts +++ b/packages/trpc/src/router/cloud-workspace/reap.ts @@ -4,7 +4,6 @@ import { eq } from "drizzle-orm"; import { logControl } from "../../lib/control-log"; import { detachAll, reapOrphanFiles } from "../../lib/files"; import { - deleteSandbox, describeSandbox, rerunBoxStartHook, runBoxArchiveHook, @@ -12,18 +11,31 @@ import { stopSandbox, } from "../../lib/sandbox"; import { publishCloudWorkspaceJob } from "./jobs"; -import { queueSettleWorkspace, settleWorkspaceNow } from "./meter"; -import { recordBootEnd, recordBoxStopped } from "./sandbox-sleep"; +import { queueSettleWorkspace } from "./meter"; +import { recordBoxStopped } from "./sandbox-sleep"; /** How long an archived workspace keeps its box running, so an undo finds it live. */ export const ARCHIVE_STOP_DELAY_SECONDS = 60; /** How long an archived workspace keeps its stopped box for an unarchive to resume. */ export const ARCHIVE_GRACE_SECONDS = 7 * 24 * 60 * 60; +/** + * Past it an unarchive is refused, even before the cleanup job deletes the + * box. Rows archived before `deleted_at` existed count from their last write. + */ +export function archiveWaitIsOver( + row: { deletedAt: Date | null; updatedAt: Date }, + now = Date.now(), +): boolean { + const archivedAt = row.deletedAt ?? row.updatedAt; + return now - archivedAt.getTime() >= ARCHIVE_GRACE_SECONDS * 1000; +} + export interface ReapArchivedCloudWorkspaceInput { cloudWorkspaceId: string; /** The archive this reap belongs to; a later archive queues its own. */ archivedAt: string; + /** `delete` only arrives from archives queued before the cleanup job took the delete over. */ stage: "stop" | "delete"; } @@ -83,14 +95,16 @@ async function runArchiveHook( export async function reapArchivedCloudWorkspace( input: ReapArchivedCloudWorkspaceInput, -): Promise<"stopped" | "reaped" | "skipped"> { +): Promise<"stopped" | "skipped"> { const row = await findRow(input.cloudWorkspaceId); if (!isStillArchived(row, input)) { return "skipped"; } if (input.stage === "delete") { - await deleteArchivedBox(row); - return "reaped"; + console.log( + `[cloud-workspace] ${row.id} reap delete left to the cleanup job (rule 1)`, + ); + return "skipped"; } if (!(await runArchiveHook(row, input))) return "skipped"; const stopped = await stopSandbox(row.providerSandboxId); @@ -109,44 +123,6 @@ export async function reapArchivedCloudWorkspace( return "stopped"; } -async function deleteArchivedBox(row: CloudWorkspaceRow): Promise { - await recordBootEnd( - { cloudWorkspaceId: row.id, organizationId: row.organizationId }, - { sessionId: null, reason: "archived" }, - ); - await settleWorkspaceNow(row.id).catch((error) => - console.error(`[cloud-workspace] ${row.id} final settle failed`, error), - ); - await deleteSandbox(row.providerSandboxId); - await reapWorkspaceFiles(row.id); - logControl("archive.deleted", { - cloudWorkspaceId: row.id, - organizationId: row.organizationId, - }); -} - -/** - * The delete stage for an archived row with no archive date: no reap was - * ever queued for it, so it has no `archivedAt` to match. Deletes only while - * the row is still archived, undated, and unchanged since `archivedBefore`. - */ -export async function reapUndatedArchivedCloudWorkspace(input: { - cloudWorkspaceId: string; - archivedBefore: Date; -}): Promise<"reaped" | "skipped"> { - const row = await findRow(input.cloudWorkspaceId); - if ( - row?.status !== "archived" || - row.deletedAt !== null || - row.provider !== "vercel" || - row.updatedAt >= input.archivedBefore - ) { - return "skipped"; - } - await deleteArchivedBox(row); - return "reaped"; -} - /** Drops the files sent to a workspace whose box is gone for good. */ export async function reapWorkspaceFiles(cloudWorkspaceId: string) { await reapOrphanFiles( @@ -175,9 +151,8 @@ function queueReapStage( } /** - * Stops the box after a short delay, and deletes it once the grace period is over. - * Throws only when the delete could not be queued; a stop that could not be - * queued happens now instead. + * Stops the box after a short delay; a stop that could not be queued happens + * now instead. The cleanup job deletes the box once the grace period is over. */ export async function queueReap( input: Omit, @@ -187,7 +162,6 @@ export async function queueReap( sandboxSessionId: string | null; }, ): Promise { - await queueReapStage({ ...input, stage: "delete" }, ARCHIVE_GRACE_SECONDS); await queueReapStage( { ...input, stage: "stop" }, ARCHIVE_STOP_DELAY_SECONDS, diff --git a/packages/trpc/src/router/cloud-workspace/retention.test.ts b/packages/trpc/src/router/cloud-workspace/retention.test.ts index ed0e34d2523..25b5883f0f5 100644 --- a/packages/trpc/src/router/cloud-workspace/retention.test.ts +++ b/packages/trpc/src/router/cloud-workspace/retention.test.ts @@ -1,42 +1,34 @@ import { describe, expect, test } from "bun:test"; -import { db } from "@superset/db/client"; +import { PgDialect } from "drizzle-orm/pg-core"; import { stub } from "../../../test/stub"; -import type { ListedSandbox } from "../../lib/sandbox"; +import type { ListedDrive, ListedSandbox } from "../../lib/sandbox"; +import * as meter from "./meter"; +import * as reap from "./reap"; import { - ARCHIVED_BACKSTOP_AFTER_MS, - findStranded, - ORPHAN_BATCH, - RETENTION_IDLE_MS, + type EnvironmentGoldens, + LEFTOVER_AGE_MS, + RETIRE_BATCH, type RetentionDeps, RUN_BUDGET_MS, reapCloudWorkspaceRetention, - retireIdleWorkspace, - retireStrandedWorkspace, - STRANDED_BATCH, - STRANDED_PROVISIONING_MS, - type StrandedWorkspace, + retireWorkspace, TEARDOWN_SPACING_MS, + WAIT_MS, + type WaitingWorkspace, + type WaitStatus, + waitOverWhere, } from "./retention"; +import * as sandboxSleep from "./sandbox-sleep"; import * as teardown from "./teardown"; -import * as transition from "./transition"; -const NOW = Date.parse("2026-11-18T06:00:00Z"); +const NOW = Date.parse("2026-11-19T06:00:00Z"); const DAY = 24 * 60 * 60 * 1000; +const BUNDLE = "0123456789abcdef".repeat(4); -const idleRow = (n: number) => ({ +const row = (n: number, status: WaitStatus = "asleep"): WaitingWorkspace => ({ id: `ws_${n}`, organizationId: "org_1", providerSandboxId: `ws-box${n}`, -}); - -const strandedRow = ( - id: string, - providerSandboxId: string, - status: StrandedWorkspace["status"], -): StrandedWorkspace => ({ - id, - organizationId: "org_1", - providerSandboxId, status, }); @@ -47,406 +39,404 @@ const box = ( name, kind: "workspace", createdAt: NOW - 60 * DAY, - statusUpdatedAt: NOW - 40 * DAY, status: "stopped", + mounts: [], + ...overrides, +}); + +const drive = ( + name: string, + overrides: Partial = {}, +): ListedDrive => ({ + name, + createdAt: NOW - 60 * DAY, + attachedTo: undefined, ...overrides, }); +const render = (where: Parameters[0]) => + new PgDialect().sqlToQuery(where); + function fakeDeps(overrides: Partial = {}) { - const queued: string[] = []; - const paused: number[] = []; + const tornDown: string[] = []; + const deletedDrives: string[] = []; const retired: string[] = []; - const reaped: unknown[] = []; + const paused: number[] = []; + const lines: string[] = []; const deps: RetentionDeps = { - findIdleWorkspaces: async () => [], - retireIdleWorkspace: async (row) => { + boxesRunningNow: async () => [], + listSandboxes: async () => [box("ws-live")], + listDrives: async () => [], + findPastWait: async () => [], + retireWorkspace: async (row) => { retired.push(row.id); return true; }, - boxIsRunning: async () => false, - listAllSandboxes: async () => [], - namedSandboxes: async () => new Set(), - countArchivedPastGrace: async () => 0, - findArchivedPastGraceAmong: async () => [], - reapArchived: async (input) => { - reaped.push(input); - return "reaped"; - }, - reapUndatedArchived: async (input) => { - reaped.push(input); - return "reaped"; - }, - findStranded: async () => [], - retireStrandedWorkspace: async (row) => { - retired.push(row.id); - return true; + listEnvironmentGoldens: async () => [], + namedByLiveRow: async () => new Set(), + namedByAnyRow: async (names) => new Set(names), + namedDrives: async () => new Set(), + currentBundleSha: async () => BUNDLE, + tearDownBox: async (name) => { + tornDown.push(name); }, - tearDownOrphan: async (name) => { - queued.push(name); + deleteDrive: async (name) => { + deletedDrives.push(name); }, pause: async (ms) => { paused.push(ms); }, + log: (line) => lines.push(line), ...overrides, }; - return { deps, queued, paused, retired, reaped }; + const decisions = () => + lines + .filter((line) => line.startsWith("[cleanup] {")) + .map((line) => JSON.parse(line.slice("[cleanup] ".length))); + return { deps, tornDown, deletedDrives, retired, paused, decisions }; } -describe("reapCloudWorkspaceRetention", () => { - test("retires each idle workspace, a pause apart", async () => { - let cutoff: Date | undefined; - const { deps, paused, retired } = fakeDeps({ - findIdleWorkspaces: async (at) => { - cutoff = at; - return [idleRow(1), idleRow(2)]; +const run = (deps: RetentionDeps, clock?: () => number) => + reapCloudWorkspaceRetention({ now: NOW, production: true, deps, clock }); + +describe("the cleanup job", () => { + test("runs only in production", async () => { + let listed = false; + const { deps } = fakeDeps({ + listSandboxes: async () => { + listed = true; + return []; }, }); - const run = await reapCloudWorkspaceRetention({ - now: NOW, - production: false, - deps, - }); - expect(cutoff?.getTime()).toBe(NOW - RETENTION_IDLE_MS); - expect(retired).toEqual(["ws_1", "ws_2"]); - expect(paused).toEqual([TEARDOWN_SPACING_MS, TEARDOWN_SPACING_MS]); - expect(run.idle).toMatchObject({ found: 2, retired: 2 }); + expect( + await reapCloudWorkspaceRetention({ now: NOW, production: false, deps }), + ).toEqual({ skipped: "not production" }); + expect(listed).toBe(false); }); - test("a box that runs a session now is skipped, not retired", async () => { - const { deps, retired } = fakeDeps({ - findIdleWorkspaces: async () => [idleRow(1)], - boxIsRunning: async () => true, - }); - const run = await reapCloudWorkspaceRetention({ - now: NOW, - production: false, - deps, + test("fails closed: an incomplete listing deletes nothing", async () => { + let expected: readonly string[] = []; + const { deps, retired, tornDown, deletedDrives } = fakeDeps({ + boxesRunningNow: async () => ["ws-running"], + listSandboxes: async (names) => { + expected = names; + throw new Error("listing lacks ws-running"); + }, + findPastWait: async () => [row(1)], }); + expect(await run(deps)).toMatchObject({ skipped: "listing incomplete" }); + expect(expected).toEqual(["ws-running"]); expect(retired).toEqual([]); - expect(run.idle.skippedRunning).toBe(1); + expect(tornDown).toEqual([]); + expect(deletedDrives).toEqual([]); }); - test("a row woken or changed after it was read keeps its box", async () => { - const { deps, paused } = fakeDeps({ - findIdleWorkspaces: async () => [idleRow(1)], - retireIdleWorkspace: async () => false, + test("rule 1: retires every row one select finds past its wait, a pause apart, and logs each", async () => { + let asked: Parameters[0] | undefined; + const { deps, retired, paused, tornDown, decisions } = fakeDeps({ + listSandboxes: async () => [box("ws-box2"), box("ws-box3")], + namedByAnyRow: async () => new Set(), + findPastWait: async (args) => { + asked = args; + return [row(1, "asleep"), row(2, "archived"), row(3, "failed")]; + }, }); - const run = await reapCloudWorkspaceRetention({ + const result = await run(deps); + expect(asked).toEqual({ now: NOW, - production: false, - deps, + listedNames: ["ws-box2", "ws-box3"], + limit: RETIRE_BATCH, + }); + expect(retired).toEqual(["ws_1", "ws_2", "ws_3"]); + expect(paused.slice(0, 3)).toEqual([ + TEARDOWN_SPACING_MS, + TEARDOWN_SPACING_MS, + TEARDOWN_SPACING_MS, + ]); + expect(tornDown).toEqual([]); + expect(decisions().filter((d) => d.rule === 1)).toEqual([ + expect.objectContaining({ + action: "delete", + reason: "asleep for more than 30 days", + }), + expect.objectContaining({ + action: "delete", + reason: "archived for more than 7 days", + }), + expect.objectContaining({ + action: "delete", + reason: "failed for more than 30 days", + }), + ]); + expect(result).toMatchObject({ + rule1: { + retired: { asleep: 1, archived: 1, failed: 1, provisioning: 0 }, + }, }); - expect(paused).toEqual([]); - expect(run.idle.skippedChanged).toBe(1); }); - test("queues only old, settled boxes no row names, and not one it just retired", async () => { - const { deps, queued } = fakeDeps({ - findIdleWorkspaces: async () => [idleRow(1)], - listAllSandboxes: async () => [ - box("ws-box1"), - box("ws-orphan"), - box("ws-named"), - box("ws-young", { createdAt: NOW - DAY }), - box("ws-recent", { statusUpdatedAt: NOW - DAY }), - box("ws-running", { status: "running" }), - box("env-orphan", { kind: "environment" }), - box("someone-elses", { kind: undefined }), - ], - namedSandboxes: async () => new Set(["ws-named"]), - }); - const run = await reapCloudWorkspaceRetention({ - now: NOW, - production: true, - deps, + test("rule 1: an asleep row whose box runs now is kept", async () => { + const { deps, retired } = fakeDeps({ + listSandboxes: async () => [box("ws-box1", { status: "running" })], + findPastWait: async () => [row(1)], }); - expect(queued).toEqual(["ws-orphan", "env-orphan"]); - expect(run.orphans).toEqual({ - listed: 7, - orphaned: 2, - queued: 2, - deferred: 0, - failed: 0, + expect(await run(deps)).toMatchObject({ rule1: { skippedRunning: 1 } }); + expect(retired).toEqual([]); + }); + + test("rule 1: a wake that lands first wins, and the row keeps its box", async () => { + const { deps, decisions } = fakeDeps({ + findPastWait: async () => [row(1)], + retireWorkspace: async () => false, }); + expect(await run(deps)).toMatchObject({ rule1: { skippedChanged: 1 } }); + expect(decisions()).toContainEqual( + expect.objectContaining({ rule: 1, action: "keep" }), + ); }); - test("outside production it touches no archived box and looks for no orphan", async () => { - let listed = false; - const { deps, reaped } = fakeDeps({ - listAllSandboxes: async () => { - listed = true; - return [box("ws-orphan")]; + test("stops starting work when its time is up and leaves the rest for the next run", async () => { + let time = 0; + const { deps, retired } = fakeDeps({ + findPastWait: async () => [row(1), row(2), row(3)], + retireWorkspace: async (row) => { + retired.push(row.id); + time += RUN_BUDGET_MS + 1; + return true; }, - findArchivedPastGraceAmong: async () => [ - { id: "ws_a", providerSandboxId: "ws-a", deletedAt: new Date(0) }, - ], }); - const run = await reapCloudWorkspaceRetention({ - now: NOW, - production: false, - deps, + expect(await run(deps, () => time)).toMatchObject({ + rule1: { deferred: 2 }, }); - expect(listed).toBe(false); - expect(reaped).toEqual([]); - expect(run.archived).toEqual({ skipped: "not production" }); - expect(run.stranded).toEqual({ skipped: "not production" }); - expect(run.orphans).toEqual({ skipped: "not production" }); + expect(retired).toEqual(["ws_1"]); }); - test("retires failed and stranded rows that still name a box, a pause apart, and does not queue their boxes again as orphans", async () => { - let asked: Parameters[0] | undefined; - const { deps, retired, queued, paused } = fakeDeps({ - listAllSandboxes: async () => [box("ws-failed"), box("ws-stranded")], - findStranded: async (args) => { - asked = args; - return [ - strandedRow("ws_f", "ws-failed", "failed"), - strandedRow("ws_p", "ws-stranded", "provisioning"), - ]; + test("rule 2: an archived environment's golden goes; a live one keeps its active golden and newest others", async () => { + const envs: EnvironmentGoldens[] = [ + { id: "env_a", archived: true, activeRef: "env-a", goldens: ["env-a"] }, + { + id: "env_l", + archived: false, + activeRef: "env-l3", + goldens: ["env-l5", "env-l4", "env-l3", "env-l2", "env-l1"], }, + ]; + const { deps, tornDown } = fakeDeps({ + listSandboxes: async () => + ["env-a", "env-l5", "env-l4", "env-l3", "env-l2", "env-l1"].map( + (name) => box(name, { kind: "environment" }), + ), + listEnvironmentGoldens: async () => envs, }); - const run = await reapCloudWorkspaceRetention({ - now: NOW, - production: true, - deps, - }); - expect(asked).toEqual({ - failedCutoff: new Date(NOW - RETENTION_IDLE_MS), - provisioningCutoff: new Date(NOW - STRANDED_PROVISIONING_MS), - names: ["ws-failed", "ws-stranded"], - limit: STRANDED_BATCH, - }); - expect(retired).toEqual(["ws_f", "ws_p"]); - expect(paused).toEqual([TEARDOWN_SPACING_MS, TEARDOWN_SPACING_MS]); - expect(queued).toEqual([]); - expect(run.stranded).toEqual({ - found: 2, - retired: 2, - skippedChanged: 0, - deferred: 0, - failed: 0, - }); + const result = await run(deps); + expect(tornDown).toEqual(["env-a", "env-l2", "env-l1"]); + expect(result).toMatchObject({ rule2: { deleted: 3, kept: 3 } }); }); - test("a failed or stranded row that changed or could not be retired is counted, and the next run tries again", async () => { - const { deps } = fakeDeps({ - findStranded: async () => [ - strandedRow("ws_f", "ws-failed", "failed"), - strandedRow("ws_p", "ws-stranded", "provisioning"), + test("rule 2: a golden already gone from the listing is not deleted again, and one a live row names is kept", async () => { + const { deps, tornDown } = fakeDeps({ + listSandboxes: async () => [box("env-shared")], + listEnvironmentGoldens: async () => [ + { + id: "e1", + archived: true, + activeRef: "env-gone", + goldens: ["env-gone"], + }, + { + id: "e2", + archived: true, + activeRef: "env-shared", + goldens: ["env-shared"], + }, ], - retireStrandedWorkspace: async (row) => { - if (row.status === "failed") return false; - throw new Error("db down"); - }, - }); - const run = await reapCloudWorkspaceRetention({ - now: NOW, - production: true, - deps, - }); - expect(run.stranded).toMatchObject({ - retired: 0, - skippedChanged: 1, - failed: 1, + namedByLiveRow: async () => new Set(["env-shared"]), }); + await run(deps); + expect(tornDown).toEqual([]); }); - test("deletes an archived box still there past its grace through the archive's own delete, by the guard that fits the row", async () => { - let cutoff: Date | undefined; - let askedAbout: string[] = []; - const archivedAt = new Date("2026-10-01T00:00:00.000Z"); - const { deps, reaped } = fakeDeps({ - listAllSandboxes: async () => [box("ws-dated"), box("ws-undated")], - countArchivedPastGrace: async (at) => { - cutoff = at; - return 40; - }, - findArchivedPastGraceAmong: async (_at, names) => { - askedAbout = names; - return [ - { id: "ws_d", providerSandboxId: "ws-dated", deletedAt: archivedAt }, - { id: "ws_u", providerSandboxId: "ws-undated", deletedAt: null }, - ]; - }, - }); - const run = await reapCloudWorkspaceRetention({ - now: NOW, - production: true, - deps, - }); - expect(cutoff?.getTime()).toBe(NOW - ARCHIVED_BACKSTOP_AFTER_MS); - expect(askedAbout).toEqual(["ws-dated", "ws-undated"]); - expect(reaped).toEqual([ - { - cloudWorkspaceId: "ws_d", - archivedAt: archivedAt.toISOString(), - stage: "delete", - }, - { cloudWorkspaceId: "ws_u", archivedBefore: cutoff }, - ]); - expect(run.archived).toEqual({ - found: 40, - alreadyDeleted: 38, - deleted: 2, - skipped: 0, - deferred: 0, - failed: 0, + test("rule 3: a box no production row names goes once it is 7 days old", async () => { + const { deps, tornDown, decisions } = fakeDeps({ + listSandboxes: async () => [ + box("ws-named"), + box("ws-orphan", { kind: undefined, status: "running" }), + box("ws-young", { createdAt: NOW - LEFTOVER_AGE_MS + DAY }), + ], + namedByAnyRow: async () => new Set(["ws-named"]), }); + const result = await run(deps); + expect(tornDown).toEqual(["ws-orphan"]); + expect(decisions()).toContainEqual( + expect.objectContaining({ + rule: 3, + name: "ws-young", + action: "keep", + reason: "made less than 7 days ago", + }), + ); + expect(result).toMatchObject({ rule3: { boxes: { deleted: 1, kept: 2 } } }); }); - test("an archived row unarchived or changed before its delete is counted as skipped", async () => { - const { deps } = fakeDeps({ - listAllSandboxes: async () => [box("ws-dated")], - countArchivedPastGrace: async () => 1, - findArchivedPastGraceAmong: async () => [ - { id: "ws_d", providerSandboxId: "ws-dated", deletedAt: new Date(0) }, + test("rule 3: a drive goes once no row names it, no box mounts it and it is 7 days old", async () => { + const { deps, deletedDrives } = fakeDeps({ + listSandboxes: async () => [ + box("ws-stopped", { mounts: ["drive-mounted"] }), ], - reapArchived: async () => "skipped", + listDrives: async () => [ + drive("drive-orphan"), + drive("drive-mounted"), + drive("drive-named"), + drive("drive-attached", { attachedTo: "ws-other" }), + drive("drive-young", { createdAt: NOW - DAY }), + ], + namedDrives: async () => new Set(["drive-named"]), }); - const run = await reapCloudWorkspaceRetention({ - now: NOW, - production: true, - deps, + const result = await run(deps); + expect(deletedDrives).toEqual(["drive-orphan"]); + expect(result).toMatchObject({ + rule3: { drives: { deleted: 1, kept: 4 } }, }); - expect(run.archived).toMatchObject({ deleted: 0, skipped: 1 }); }); - test("queues at most a batch of orphans a run and counts the rest", async () => { - const many = Array.from({ length: ORPHAN_BATCH + 3 }, (_, i) => - box(`ws-orphan${i}`), - ); - const { deps, queued } = fakeDeps({ - listAllSandboxes: async () => many, + test("rule 3: an agents drive goes once it is not the current release's and no box, stopped ones included, mounts it", async () => { + const old = "aaaaaaaaaaaa"; + const { deps, deletedDrives } = fakeDeps({ + listSandboxes: async () => [ + box("ws-stopped", { mounts: [`agent-tools-iad1-${old}`] }), + ], + listDrives: async () => [ + drive(`agent-tools-sfo1-${BUNDLE.slice(0, 12)}`), + drive(`agent-tools-sfo1-${old}`), + drive(`agent-tools-iad1-${old}`), + drive("agent-tools-sfo1-bbbbbbbbbbbb", { createdAt: NOW - DAY / 2 }), + ], }); - const run = await reapCloudWorkspaceRetention({ - now: NOW, - production: true, - deps, + const result = await run(deps); + expect(deletedDrives).toEqual([`agent-tools-sfo1-${old}`]); + expect(result).toMatchObject({ + rule3: { agentsDrives: { deleted: 1, kept: 3 } }, }); - expect(queued).toHaveLength(ORPHAN_BATCH); - expect(run.orphans).toMatchObject({ queued: ORPHAN_BATCH, deferred: 3 }); }); - test("stops starting work when its time is up and leaves the rest for the next run", async () => { - let time = 0; - const { deps, retired } = fakeDeps({ - findIdleWorkspaces: async () => [idleRow(1), idleRow(2), idleRow(3)], - retireIdleWorkspace: async (row) => { - retired.push(row.id); - time += RUN_BUDGET_MS + 1; - return true; - }, - }); - const run = await reapCloudWorkspaceRetention({ - now: NOW, - production: false, - deps, - clock: () => time, + test("rule 3: no current release recorded keeps every agents drive", async () => { + const { deps, deletedDrives } = fakeDeps({ + listDrives: async () => [drive("agent-tools-sfo1-aaaaaaaaaaaa")], + currentBundleSha: async () => null, }); - expect(retired).toEqual(["ws_1"]); - expect(run.idle).toMatchObject({ retired: 1, deferred: 2 }); + await run(deps); + expect(deletedDrives).toEqual([]); }); }); -describe("retireIdleWorkspace", () => { +describe("retireWorkspace", () => { let retiring: Parameters | undefined; - let retires = true; - let failed: Parameters[0] | undefined; + let steps: string[] = []; stub(teardown, { retireAndTearDown: async ( ...args: Parameters ) => { retiring = args; - return retires; + steps.push("retire"); + return true; }, }); - stub(transition, { - failCloudWorkspace: async ( - args: Parameters[0], - ) => { - failed = args; - return true; + stub(sandboxSleep, { + recordBootEnd: async () => { + steps.push("end boots"); + }, + }); + stub(meter, { + settleWorkspaceNow: async () => { + steps.push("settle"); + }, + }); + stub(reap, { + reapWorkspaceFiles: async () => { + steps.push("files"); }, }); - test("retires the box name while the row is asleep and idle, then fails the row as an expired snapshot", async () => { - const cutoff = new Date(NOW - RETENTION_IDLE_MS); - retires = false; - expect(await retireIdleWorkspace(idleRow(1), cutoff)).toBe(false); - expect(failed).toBeUndefined(); - retires = true; - expect(await retireIdleWorkspace(idleRow(1), cutoff)).toBe(true); + async function retireAs(status: WaitStatus) { + retiring = undefined; + steps = []; + expect(await retireWorkspace(row(1, status), NOW)).toBe(true); expect(retiring?.[0]).toEqual({ id: "ws_1", providerSandboxId: "ws-box1", driveName: null, }); - expect(retiring?.[1]).toBeDefined(); - expect(failed).toMatchObject({ - scope: { cloudWorkspaceId: "ws_1", organizationId: "org_1" }, - from: ["asleep"], - reason: "snapshot_expired", - expected: true, - set: { sandboxUrl: null }, + return { + condition: render( + retiring?.[1] as Parameters[0], + ), + set: retiring?.[2], + }; + } + + const cutoff = (status: WaitStatus) => + new Date(NOW - WAIT_MS[status]).toISOString(); + + test("asleep: 30 days after last use, the one write retires it as an expired snapshot", async () => { + const { condition, set } = await retireAs("asleep"); + expect(condition.params).toEqual( + expect.arrayContaining(["asleep", cutoff("asleep")]), + ); + expect(condition.sql).toContain('"sandbox_expires_at"'); + expect(set).toMatchObject({ + status: "failed", + failureReason: "snapshot_expired", + sandboxUrl: null, }); }); - test("a stranded provisioning row gives up its box and fails; a failed row only gives up its box", async () => { - const cutoffs = { - failed: new Date(NOW - RETENTION_IDLE_MS), - provisioning: new Date(NOW - STRANDED_PROVISIONING_MS), - }; - retires = true; - failed = undefined; - expect( - await retireStrandedWorkspace( - strandedRow("ws_f", "ws-failed", "failed"), - cutoffs, - ), - ).toBe(true); - expect(retiring?.[0]).toEqual({ - id: "ws_f", - providerSandboxId: "ws-failed", - driveName: null, - }); - expect(failed).toBeUndefined(); - expect( - await retireStrandedWorkspace( - strandedRow("ws_p", "ws-stranded", "provisioning"), - cutoffs, - ), - ).toBe(true); - expect(failed).toMatchObject({ - scope: { cloudWorkspaceId: "ws_p" }, - from: ["provisioning"], - reason: "create_failed", - expected: true, + test("archived: 7 days after the archive, it ends its boots and settles, then retires and drops its files", async () => { + const { condition, set } = await retireAs("archived"); + expect(condition.params).toEqual( + expect.arrayContaining(["archived", cutoff("archived")]), + ); + expect(condition.sql).toContain('"deleted_at"'); + expect(set).toEqual({}); + expect(steps).toEqual(["end boots", "settle", "retire", "files"]); + }); + + test("failed: 30 days after it failed, it is retired and stays failed", async () => { + const { condition, set } = await retireAs("failed"); + expect(condition.params).toEqual( + expect.arrayContaining(["failed", cutoff("failed")]), + ); + expect(condition.sql).toContain('"failed_at"'); + expect(set).toEqual({}); + }); + + test("stuck creating: a day with no boot, it is retired as a failed create", async () => { + const { condition, set } = await retireAs("provisioning"); + expect(condition.params).toEqual( + expect.arrayContaining(["provisioning", cutoff("provisioning")]), + ); + expect(condition.sql).toContain('"started_at"'); + expect(set).toMatchObject({ + status: "failed", + failureReason: "create_failed", }); }); }); -describe("findStranded", () => { - let queries = 0; - const chain = { - from: () => chain, - where: () => chain, - orderBy: () => chain, - limit: async () => { - queries++; - return []; - }, - }; - stub(db, { select: () => chain }); - - test("looks for failed rows only among listed boxes, so a retired name leaves the selection", async () => { - const args = { - failedCutoff: new Date(NOW - RETENTION_IDLE_MS), - provisioningCutoff: new Date(NOW - STRANDED_PROVISIONING_MS), - limit: STRANDED_BATCH, - }; - await findStranded({ ...args, names: [] }); - expect(queries).toBe(1); - await findStranded({ ...args, names: ["ws-a"] }); - expect(queries).toBe(3); +describe("waitOverWhere", () => { + test("a retired row leaves the selection: by its new status, or, keeping its status, because its new name is not listed", () => { + const listed = render(waitOverWhere({ now: NOW, listedNames: ["ws-a"] })); + expect(listed.params).toEqual( + expect.arrayContaining([ + "asleep", + "archived", + "failed", + "provisioning", + "ws-a", + ]), + ); + expect(listed.sql.match(/string_to_array/g)).toHaveLength(2); + const none = render(waitOverWhere({ now: NOW, listedNames: [] })); + expect(none.sql).not.toContain("string_to_array"); + expect(none.sql.match(/false/g)).toHaveLength(2); }); }); diff --git a/packages/trpc/src/router/cloud-workspace/retention.ts b/packages/trpc/src/router/cloud-workspace/retention.ts index 080df23a957..db089842384 100644 --- a/packages/trpc/src/router/cloud-workspace/retention.ts +++ b/packages/trpc/src/router/cloud-workspace/retention.ts @@ -1,67 +1,83 @@ import * as Sentry from "@sentry/core"; import { db } from "@superset/db/client"; -import { cloudWorkspaceBoots, cloudWorkspaces } from "@superset/db/schema"; +import type { CloudWorkspaceFailureReason } from "@superset/db/enums"; +import { + cloudWorkspaceBoots, + cloudWorkspaces, + environments, + sandboxBundles, +} from "@superset/db/schema"; +import { SANDBOX_AGENT_TOOLS_DRIVE_PATTERN } from "@superset/shared/sandbox/contract"; import { and, asc, - count, + desc, eq, - inArray, - isNotNull, + gt, isNull, lt, or, type SQL, sql, } from "drizzle-orm"; +import { logControl } from "../../lib/control-log"; +import { nudge } from "../../lib/realtime"; import { - describeSandbox, + deleteDrive, + type ListedDrive, type ListedSandbox, - listAllSandboxes, - SandboxUnavailableError, + listProjectDrives, + listProjectSandboxes, } from "../../lib/sandbox"; -import { - ARCHIVE_GRACE_SECONDS, - reapArchivedCloudWorkspace, - reapUndatedArchivedCloudWorkspace, -} from "./reap"; +import { goldensToDelete } from "./golden-rotation"; +import { settleWorkspaceNow } from "./meter"; +import { ARCHIVE_GRACE_SECONDS, reapWorkspaceFiles } from "./reap"; +import { recordBootEnd } from "./sandbox-sleep"; import { deleteSandboxAndDriveLater, retireAndTearDown } from "./teardown"; -import { namedSandboxes } from "./teardown-guard"; -import { failCloudWorkspace } from "./transition"; +import { namedByAnyRow, namedSandboxes } from "./teardown-guard"; export const RETENTION_JOB_PATH = "/api/cloud-workspaces/jobs/reap-retention"; const DAY_MS = 24 * 60 * 60 * 1000; -/** What Vercel's snapshot expiry gave customers before we owned it. */ -export const RETENTION_IDLE_MS = 30 * DAY_MS; -/** Longer than a release or a promote takes to write the name of a golden it built. */ -export const ORPHAN_MIN_AGE_MS = 7 * DAY_MS; + +/** Rule 1: how long a workspace not in use keeps its box, snapshot and drive. */ +export const WAIT_MS = { + /** Since last use: what Vercel's snapshot expiry gave before we owned it. */ + asleep: 30 * DAY_MS, + /** Since the archive; an unarchive inside it keeps everything, after it is refused. */ + archived: ARCHIVE_GRACE_SECONDS * 1000, + /** Since it failed: long enough to look at what went wrong. */ + failed: 30 * DAY_MS, + /** Stuck creating with no boot since: every delivery of its job died. */ + provisioning: DAY_MS, +} as const; + +export type WaitStatus = keyof typeof WAIT_MS; +const WAIT_STATUSES = Object.keys(WAIT_MS) as WaitStatus[]; + +/** Rule 3: a box, drive or golden nothing in production names goes this long after it was made. */ +export const LEFTOVER_AGE_MS = 7 * DAY_MS; +/** A release publishes its agents drives before it records its bundle as current. */ +export const NEW_AGENTS_DRIVE_MS = DAY_MS; + /** * Every 6 hours, so 800 a day: production made a median 156 workspaces a day * and at most 725 in the 30 days to 2026-11-18. A backlog takes more runs. */ -export const REAP_BATCH = 200; -export const ORPHAN_BATCH = 200; -/** Longer than any provision takes, its QStash retries included: minutes. */ -export const STRANDED_PROVISIONING_MS = DAY_MS; -/** Failed and stranded rows are rare: 2 failed and none stranded on 2026-11-19. */ -export const STRANDED_BATCH = 50; -/** A backstop for a lost archive delete, which deletes inline: few a run. */ -export const ARCHIVED_BATCH = 20; -/** The archive's own delete runs at the grace; a day later it has had its retries. */ -export const ARCHIVED_BACKSTOP_AFTER_MS = ARCHIVE_GRACE_SECONDS * 1000 + DAY_MS; +export const RETIRE_BATCH = 200; +export const GOLDEN_BATCH = 20; +export const LEFTOVER_BOX_BATCH = 200; +export const DRIVE_BATCH = 50; /** Stops starting new work here, well inside the route's 300 s; the rest waits for the next run. */ export const RUN_BUDGET_MS = 200_000; /** At most 4 teardowns queued a second: far under Vercel's 20 deletions a second per team. */ export const TEARDOWN_SPACING_MS = 250; const NAME_LOOKUP_CHUNK = 500; -const SETTLED_STATUSES = new Set(["stopped", "failed", "aborted"]); +/** Boxes running now, which the box listing must contain to count as whole. */ +const EXPECTED_IN_LISTING = 3; +const RUNNING_STATUSES = new Set(["pending", "running"]); -export interface IdleWorkspace { - id: string; - organizationId: string; - providerSandboxId: string; -} +const at = (cutoff: Date) => sql`${cutoff.toISOString()}::timestamptz`; /** * Nothing used the workspace since `cutoff`. Boots count because a wake @@ -70,352 +86,315 @@ export interface IdleWorkspace { * with the boots, the later one wins and the box is kept longer. */ export function idleSince(cutoff: Date): SQL { - const at = sql`${cutoff.toISOString()}::timestamptz`; return and( lt(cloudWorkspaces.createdAt, cutoff), or( isNull(cloudWorkspaces.sandboxExpiresAt), lt(cloudWorkspaces.sandboxExpiresAt, cutoff), ), - sql`not exists (select 1 from ${cloudWorkspaceBoots} where ${cloudWorkspaceBoots.cloudWorkspaceId} = ${cloudWorkspaces.id} and (${cloudWorkspaceBoots.stoppedAt} is null or ${cloudWorkspaceBoots.stoppedAt} >= ${at} or ${cloudWorkspaceBoots.startedAt} >= ${at}))`, + sql`not exists (select 1 from ${cloudWorkspaceBoots} where ${cloudWorkspaceBoots.cloudWorkspaceId} = ${cloudWorkspaces.id} and (${cloudWorkspaceBoots.stoppedAt} is null or ${cloudWorkspaceBoots.stoppedAt} >= ${at(cutoff)} or ${cloudWorkspaceBoots.startedAt} >= ${at(cutoff)}))`, ) as SQL; } -function stillIdleAndAsleep(cutoff: Date): SQL { +/** Rows archived before `deleted_at` existed count from their last write. */ +function archivedBefore(cutoff: Date): SQL { + return sql`coalesce(${cloudWorkspaces.deletedAt}, ${cloudWorkspaces.updatedAt}) < ${at(cutoff)}`; +} + +/** Rows failed before `failed_at` existed count from their last write. */ +function failedBefore(cutoff: Date): SQL { return and( - eq(cloudWorkspaces.provider, "vercel"), isNull(cloudWorkspaces.deletedAt), - idleSince(cutoff), + sql`coalesce(${cloudWorkspaces.failedAt}, ${cloudWorkspaces.updatedAt}) < ${at(cutoff)}`, ) as SQL; } -async function findIdleWorkspaces( - cutoff: Date, - limit: number, -): Promise { - return db - .select({ - id: cloudWorkspaces.id, - organizationId: cloudWorkspaces.organizationId, - providerSandboxId: cloudWorkspaces.providerSandboxId, - }) - .from(cloudWorkspaces) - .where( - and(eq(cloudWorkspaces.status, "asleep"), stillIdleAndAsleep(cutoff)), - ) - .orderBy(asc(cloudWorkspaces.createdAt)) - .limit(limit); +/** An unarchive moves an old row to provisioning, so `updated_at` counts, not only `created_at`. */ +function noBootSince(cutoff: Date): SQL { + return and( + isNull(cloudWorkspaces.deletedAt), + lt(cloudWorkspaces.createdAt, cutoff), + lt(cloudWorkspaces.updatedAt, cutoff), + sql`not exists (select 1 from ${cloudWorkspaceBoots} where ${cloudWorkspaceBoots.cloudWorkspaceId} = ${cloudWorkspaces.id} and ${cloudWorkspaceBoots.startedAt} >= ${at(cutoff)})`, + ) as SQL; } -/** - * Takes the box's name off the row while it is still asleep and idle, and - * queues the box's teardown; then fails the row as the 410 path does. A wake - * between the two finds no box and fails the row itself. False when the row - * changed since it was read: woken, archived, or given another box. - */ -export async function retireIdleWorkspace( - row: IdleWorkspace, - cutoff: Date, -): Promise { - const retired = await retireAndTearDown( - { id: row.id, providerSandboxId: row.providerSandboxId, driveName: null }, - and(eq(cloudWorkspaces.status, "asleep"), stillIdleAndAsleep(cutoff)), - ); - if (!retired) return false; - await failCloudWorkspace({ - scope: { cloudWorkspaceId: row.id, organizationId: row.organizationId }, - from: ["asleep"], - where: stillIdleAndAsleep(cutoff), - set: { sandboxUrl: null }, - reason: "snapshot_expired", - error: new Error(`idle since before ${cutoff.toISOString()}`), - expected: true, - }); - return true; +interface WaitRule { + since: (cutoff: Date) => SQL; + /** The status the retire write gives the row; null keeps its status. */ + failAs: CloudWorkspaceFailureReason | null; } -/** True when the box runs a session now; false when it is stopped or gone. */ -async function boxIsRunning(providerSandboxId: string): Promise { - try { - return (await describeSandbox(providerSandboxId)).running; - } catch (error) { - if (error instanceof SandboxUnavailableError) return false; - throw error; - } -} +/** + * Rule 1's wait table. A retired row leaves the selection: an asleep or + * stuck row fails, and a row that keeps its status (archived, failed) is + * selected only while its box is listed, which a retired name never is. + */ +export const WAIT_RULES: Record = { + asleep: { + since: (cutoff) => + and(isNull(cloudWorkspaces.deletedAt), idleSince(cutoff)) as SQL, + failAs: "snapshot_expired", + }, + archived: { since: archivedBefore, failAs: null }, + failed: { since: failedBefore, failAs: null }, + provisioning: { since: noBootSince, failAs: "create_failed" }, +}; -export interface StrandedWorkspace { - id: string; - organizationId: string; - providerSandboxId: string; - status: "failed" | "provisioning"; +export function waitCutoff(status: WaitStatus, now: number): Date { + return new Date(now - WAIT_MS[status]); } -/** Failed before `cutoff`; rows failed before `failed_at` existed count from their last write. */ -export function failedBefore(cutoff: Date): SQL { +/** Still in `status` and still past its wait: the condition of the one retire write. */ +export function pastWait(status: WaitStatus, now: number): SQL { return and( eq(cloudWorkspaces.provider, "vercel"), - isNull(cloudWorkspaces.deletedAt), - eq(cloudWorkspaces.status, "failed"), - sql`coalesce(${cloudWorkspaces.failedAt}, ${cloudWorkspaces.updatedAt}) < ${cutoff.toISOString()}::timestamptz`, + eq(cloudWorkspaces.status, status), + WAIT_RULES[status].since(waitCutoff(status, now)), ) as SQL; } -/** - * Provisioning since before `cutoff` with no boot since: every delivery of - * its job died. An unarchive moves a row to provisioning, so `updated_at` - * counts, not only `created_at`. - */ -export function strandedSince(cutoff: Date): SQL { - return and( - eq(cloudWorkspaces.provider, "vercel"), - isNull(cloudWorkspaces.deletedAt), - eq(cloudWorkspaces.status, "provisioning"), - lt(cloudWorkspaces.createdAt, cutoff), - lt(cloudWorkspaces.updatedAt, cutoff), - sql`not exists (select 1 from ${cloudWorkspaceBoots} where ${cloudWorkspaceBoots.cloudWorkspaceId} = ${cloudWorkspaces.id} and ${cloudWorkspaceBoots.startedAt} >= ${cutoff.toISOString()}::timestamptz)`, +function amongNames(names: readonly string[]): SQL { + const usable = names.filter((name) => !name.includes(",")); + if (usable.length === 0) return sql`false`; + return sql`${cloudWorkspaces.providerSandboxId} = any(string_to_array(${usable.join(",")}, ','))`; +} + +/** Rule 1 as one select. */ +export function waitOverWhere({ + now, + listedNames, +}: { + now: number; + listedNames: readonly string[]; +}): SQL { + return or( + ...WAIT_STATUSES.map((status) => + WAIT_RULES[status].failAs + ? pastWait(status, now) + : and(pastWait(status, now), amongNames(listedNames)), + ), ) as SQL; } -const strandedColumns = { - id: cloudWorkspaces.id, - organizationId: cloudWorkspaces.organizationId, - providerSandboxId: cloudWorkspaces.providerSandboxId, - status: cloudWorkspaces.status, -}; +export interface WaitingWorkspace { + id: string; + organizationId: string; + providerSandboxId: string; + status: WaitStatus; +} -/** - * Stranded provisioning rows by their status alone: retiring fails them, so - * they leave. Failed rows only while their box is in `names`: a retired name - * is in no listing, so they leave too, and none can fill the batch forever. - */ -export async function findStranded({ - failedCutoff, - provisioningCutoff, - names, - limit, -}: { - failedCutoff: Date; - provisioningCutoff: Date; - names: string[]; +async function findPastWait(args: { + now: number; + listedNames: readonly string[]; limit: number; -}): Promise { - const found = (await db - .select(strandedColumns) +}): Promise { + const rows = await db + .select({ + id: cloudWorkspaces.id, + organizationId: cloudWorkspaces.organizationId, + providerSandboxId: cloudWorkspaces.providerSandboxId, + status: cloudWorkspaces.status, + }) .from(cloudWorkspaces) - .where(strandedSince(provisioningCutoff)) - .orderBy(asc(cloudWorkspaces.updatedAt)) - .limit(limit)) as StrandedWorkspace[]; - for ( - let i = 0; - i < names.length && found.length < limit; - i += NAME_LOOKUP_CHUNK - ) { - found.push( - ...((await db - .select(strandedColumns) - .from(cloudWorkspaces) - .where( - and( - failedBefore(failedCutoff), - inArray( - cloudWorkspaces.providerSandboxId, - names.slice(i, i + NAME_LOOKUP_CHUNK), - ), - ), - ) - .limit(limit - found.length)) as StrandedWorkspace[]), - ); - } - return found; + .where(waitOverWhere(args)) + .orderBy(asc(cloudWorkspaces.createdAt)) + .limit(args.limit); + return rows.filter((row): row is WaitingWorkspace => + (WAIT_STATUSES as string[]).includes(row.status), + ); } /** - * Takes the box name off a failed or stranded row and queues the box's - * teardown, on the status and age read. A stranded row then fails, so the - * client stops waiting for it. False when the row changed since it was read. + * teardown(row): one conditional write takes the box name off the row, and + * the status with it, while the row is still in its status and still past its + * wait; then the box is deleted through the retried teardown job. A wake that + * lands first wins, and this returns false. An archived row ends its boots + * and settles its usage first, and its files go with the box. */ -export async function retireStrandedWorkspace( - row: StrandedWorkspace, - cutoffs: { failed: Date; provisioning: Date }, +export async function retireWorkspace( + row: WaitingWorkspace, + now: number, ): Promise { - const stillStranded = - row.status === "failed" - ? failedBefore(cutoffs.failed) - : strandedSince(cutoffs.provisioning); + const scope = { + cloudWorkspaceId: row.id, + organizationId: row.organizationId, + }; + if (row.status === "archived") { + await recordBootEnd(scope, { sessionId: null, reason: "archived" }); + await settleWorkspaceNow(row.id).catch((error) => + console.error(`[cloud-workspace] ${row.id} final settle failed`, error), + ); + } + const failAs = WAIT_RULES[row.status].failAs; const retired = await retireAndTearDown( { id: row.id, providerSandboxId: row.providerSandboxId, driveName: null }, - stillStranded, + pastWait(row.status, now), + failAs + ? { + status: "failed", + failureReason: failAs, + failedAt: new Date(now), + sandboxUrl: null, + } + : {}, ); if (!retired) return false; - if (row.status === "provisioning") { - await failCloudWorkspace({ - scope: { cloudWorkspaceId: row.id, organizationId: row.organizationId }, - from: ["provisioning"], - set: { sandboxUrl: null }, - reason: "create_failed", - error: new Error( - `provisioning since before ${cutoffs.provisioning.toISOString()}`, - ), - expected: true, - }); + if (row.status === "archived") await reapWorkspaceFiles(row.id); + if (failAs) { + logControl("workspace.failed", scope, { "failure.reason": failAs }, "warn"); + nudge(row.organizationId, "cloud_workspaces"); } return true; } -function archivedPastGrace(cutoff: Date): SQL { - return and( - eq(cloudWorkspaces.provider, "vercel"), - eq(cloudWorkspaces.status, "archived"), - or( - and( - isNotNull(cloudWorkspaces.deletedAt), - lt(cloudWorkspaces.deletedAt, cutoff), - ), +/** A few boxes running now: a listing without them is not the whole project. */ +async function boxesRunningNow(now: number): Promise { + const rows = await db + .select({ name: cloudWorkspaces.providerSandboxId }) + .from(cloudWorkspaces) + .where( and( - isNull(cloudWorkspaces.deletedAt), - lt(cloudWorkspaces.updatedAt, cutoff), + eq(cloudWorkspaces.provider, "vercel"), + eq(cloudWorkspaces.status, "awake"), + gt(cloudWorkspaces.sandboxExpiresAt, new Date(now + 60_000)), ), - ), - ) as SQL; + ) + .limit(EXPECTED_IN_LISTING); + return rows.map((row) => row.name); } -async function countArchivedPastGrace(cutoff: Date): Promise { - const [row] = await db - .select({ n: count() }) - .from(cloudWorkspaces) - .where(archivedPastGrace(cutoff)); - return row?.n ?? 0; +export interface EnvironmentGoldens { + id: string; + archived: boolean; + activeRef: string | null; + /** Undeleted goldens, newest first. */ + goldens: string[]; } -export interface ArchivedWorkspace { - id: string; - providerSandboxId: string; - deletedAt: Date | null; +/** On main an environment records only its active golden; a fork's golden is a box. */ +async function listEnvironmentGoldens(): Promise { + const rows = await db + .select({ + id: environments.id, + archivedAt: environments.archivedAt, + sourceKind: environments.sourceKind, + sourceRef: environments.sourceRef, + }) + .from(environments) + .where(eq(environments.provider, "vercel")); + return rows.map((row) => ({ + id: row.id, + archived: row.archivedAt !== null, + activeRef: row.sourceKind === "fork" ? row.sourceRef : null, + goldens: row.sourceKind === "fork" ? [row.sourceRef] : [], + })); } -/** The archived rows past their grace whose box is among `names`. */ -async function findArchivedPastGraceAmong( - cutoff: Date, - names: string[], -): Promise { - const found: ArchivedWorkspace[] = []; - for (let i = 0; i < names.length; i += NAME_LOOKUP_CHUNK) { - found.push( - ...(await db - .select({ - id: cloudWorkspaces.id, - providerSandboxId: cloudWorkspaces.providerSandboxId, - deletedAt: cloudWorkspaces.deletedAt, - }) - .from(cloudWorkspaces) - .where( - and( - archivedPastGrace(cutoff), - inArray( - cloudWorkspaces.providerSandboxId, - names.slice(i, i + NAME_LOOKUP_CHUNK), - ), - ), - )), - ); - } - return found; +async function currentBundleSha(): Promise { + const [row] = await db + .select({ sha: sandboxBundles.sha }) + .from(sandboxBundles) + .orderBy(desc(sandboxBundles.releasedAt)) + .limit(1); + return row?.sha ?? null; } -async function namedAmong(names: string[]): Promise> { - const named = new Set(); +async function inChunks( + names: readonly string[], + lookup: (chunk: string[]) => Promise>, +): Promise> { + const found = new Set(); for (let i = 0; i < names.length; i += NAME_LOOKUP_CHUNK) { - for (const name of await namedSandboxes( - names.slice(i, i + NAME_LOOKUP_CHUNK), - )) { - named.add(name); + for (const name of await lookup(names.slice(i, i + NAME_LOOKUP_CHUNK))) { + found.add(name); } } - return named; + return found; } export interface RetentionDeps { - findIdleWorkspaces: typeof findIdleWorkspaces; - retireIdleWorkspace: typeof retireIdleWorkspace; - boxIsRunning: typeof boxIsRunning; - listAllSandboxes: typeof listAllSandboxes; - namedSandboxes: typeof namedAmong; - countArchivedPastGrace: typeof countArchivedPastGrace; - findArchivedPastGraceAmong: typeof findArchivedPastGraceAmong; - reapArchived: typeof reapArchivedCloudWorkspace; - reapUndatedArchived: typeof reapUndatedArchivedCloudWorkspace; - findStranded: typeof findStranded; - retireStrandedWorkspace: typeof retireStrandedWorkspace; - tearDownOrphan: (providerSandboxId: string) => Promise; + boxesRunningNow: typeof boxesRunningNow; + listSandboxes: typeof listProjectSandboxes; + listDrives: typeof listProjectDrives; + findPastWait: typeof findPastWait; + retireWorkspace: typeof retireWorkspace; + listEnvironmentGoldens: typeof listEnvironmentGoldens; + /** Named by a live workspace or an unarchived environment: what a golden delete must not take. */ + namedByLiveRow: (names: readonly string[]) => Promise>; + /** Named by any production row, archived ones included. */ + namedByAnyRow: (names: readonly string[]) => Promise>; + /** Workspace drives rows name: none on main; #9847's `drive_name` and `golden_drive_name`. */ + namedDrives: (names: readonly string[]) => Promise>; + currentBundleSha: typeof currentBundleSha; + tearDownBox: (providerSandboxId: string) => Promise; + deleteDrive: typeof deleteDrive; pause: (ms: number) => Promise; + log: (line: string) => void; } const defaultDeps: RetentionDeps = { - findIdleWorkspaces, - retireIdleWorkspace, - boxIsRunning, - listAllSandboxes, - namedSandboxes: namedAmong, - countArchivedPastGrace, - findArchivedPastGraceAmong, - reapArchived: reapArchivedCloudWorkspace, - reapUndatedArchived: reapUndatedArchivedCloudWorkspace, - findStranded, - retireStrandedWorkspace, - tearDownOrphan: (providerSandboxId) => + boxesRunningNow, + listSandboxes: listProjectSandboxes, + listDrives: listProjectDrives, + findPastWait, + retireWorkspace, + listEnvironmentGoldens, + namedByLiveRow: (names) => inChunks(names, namedSandboxes), + namedByAnyRow: (names) => inChunks(names, namedByAnyRow), + namedDrives: async () => new Set(), + currentBundleSha, + tearDownBox: (providerSandboxId) => deleteSandboxAndDriveLater({ providerSandboxId, driveName: null }), + deleteDrive, pause: (ms) => new Promise((resolve) => setTimeout(resolve, ms)), + log: (line) => console.log(line), }; -export interface RetentionRun { - idle: { - found: number; - retired: number; - skippedRunning: number; - skippedChanged: number; - deferred: number; - failed: number; - }; - archived: - | { - found: number; - alreadyDeleted: number; - deleted: number; - skipped: number; - deferred: number; - failed: number; - } - | { skipped: "not production" }; - stranded: - | { - found: number; - retired: number; +type Counts = { + kept: number; + deleted: number; + deferred: number; + failed: number; +}; + +const perStatus = (): Record => ({ + asleep: 0, + archived: 0, + failed: 0, + provisioning: 0, +}); + +export type RetentionRun = + | { skipped: "not production" } + | { skipped: "listing incomplete"; reason: string } + | { + rule1: { + found: Record; + retired: Record; + skippedRunning: number; skippedChanged: number; deferred: number; failed: number; - } - | { skipped: "not production" }; - orphans: - | { - listed: number; - orphaned: number; - queued: number; - deferred: number; - failed: number; - } - | { skipped: "not production" }; -} + }; + rule2: Counts & { environments: number }; + rule3: { boxes: Counts; drives: Counts; agentsDrives: Counts }; + }; + +type Decision = { + rule: 1 | 2 | 3; + kind: "workspace" | "box" | "golden" | "drive" | "agents drive"; + name: string; + action: "keep" | "delete"; + reason: string; +}; /** - * The one place a workspace box is deleted for age. Every 6 hours: a workspace asleep - * and unused for `RETENTION_IDLE_MS` fails as `snapshot_expired`, exactly as a - * wake that met Vercel's expiry did, and its box is torn down. Then, in - * production only: an archived box still there a day after its grace is - * deleted (its own delete was lost); a row failed for `RETENTION_IDLE_MS`, or - * stuck provisioning for `STRANDED_PROVISIONING_MS`, gives up its box; and - * boxes we tagged that no row names are torn down. Every deployment shares the Vercel project, and a dev - * database copied from production names production's boxes. + * The cleanup job, every 6 hours, production only: the one place that + * deletes for age or for being unused. Rule 1 retires workspaces past their + * wait, rule 2 deletes goldens an environment no longer keeps, rule 3 + * deletes what nothing in production names. It reads the whole Vercel + * listing first and deletes nothing if it cannot. */ export async function reapCloudWorkspaceRetention({ now = Date.now(), @@ -428,31 +407,44 @@ export async function reapCloudWorkspaceRetention({ deps?: RetentionDeps; clock?: () => number; }): Promise { + if (!production) { + deps.log("[cleanup] skipped: only production runs the cleanup job"); + return { skipped: "not production" }; + } const startedAt = clock(); const outOfTime = () => clock() - startedAt > RUN_BUDGET_MS; - const retiredNames = new Set(); - const idle = await reapIdle({ now, deps, retiredNames, outOfTime }); - const notProduction = { skipped: "not production" } as const; - const boxes = production ? await deps.listAllSandboxes() : []; - const archived = production - ? await reapArchived({ now, deps, boxes, outOfTime }) - : notProduction; - const stranded = production - ? await reapStranded({ now, deps, boxes, retiredNames, outOfTime }) - : notProduction; - const orphans = production - ? await reapOrphans({ now, deps, boxes, retiredNames, outOfTime }) - : notProduction; - const run = { idle, archived, stranded, orphans }; - console.log("[cloud-workspace] retention run", JSON.stringify(run)); + const decide = (decision: Decision) => + deps.log(`[cleanup] ${JSON.stringify(decision)}`); + let boxes: ListedSandbox[]; + let drives: ListedDrive[]; + try { + boxes = await deps.listSandboxes(await deps.boxesRunningNow(now)); + drives = await deps.listDrives(); + } catch (error) { + const reason = String(error).slice(0, 300); + deps.log(`[cleanup] listing incomplete, deleted nothing: ${reason}`); + Sentry.captureMessage("cleanup job: listing incomplete, deleted nothing", { + level: "warning", + extra: { reason }, + }); + return { skipped: "listing incomplete", reason }; + } + const ctx = { now, deps, boxes, drives, outOfTime, decide }; + const retired = new Set(); + const rule1 = await retireWorkspaces({ ...ctx, retired }); + const rule2 = await pruneGoldens({ ...ctx, retired }); + const rule3 = await deleteLeftovers({ ...ctx, retired }); + const run = { rule1, rule2, rule3 }; + deps.log(`[cleanup] run ${JSON.stringify(run)}`); Sentry.logger.info("retention.run", flatten(run)); if ( - idle.failed || - ("failed" in archived && archived.failed) || - ("failed" in stranded && stranded.failed) || - ("failed" in orphans && orphans.failed) + rule1.failed || + rule2.failed || + rule3.boxes.failed || + rule3.drives.failed || + rule3.agentsDrives.failed ) { - Sentry.captureMessage("cloud workspace retention run had failures", { + Sentry.captureMessage("cleanup job run had failures", { level: "warning", extra: { ...run }, }); @@ -460,43 +452,81 @@ export async function reapCloudWorkspaceRetention({ return run; } -async function reapIdle({ - now, - deps, - retiredNames, - outOfTime, -}: { +type Ctx = { now: number; deps: RetentionDeps; - retiredNames: Set; + boxes: ListedSandbox[]; + drives: ListedDrive[]; outOfTime: () => boolean; -}): Promise { - const cutoff = new Date(now - RETENTION_IDLE_MS); - const rows = await deps.findIdleWorkspaces(cutoff, REAP_BATCH); + decide: (decision: Decision) => void; + retired: Set; +}; + +async function retireWorkspaces({ + now, + deps, + boxes, + outOfTime, + decide, + retired, +}: Ctx) { + const status = new Map(boxes.map((box) => [box.name, box.status])); + const rows = await deps.findPastWait({ + now, + listedNames: boxes.map((box) => box.name), + limit: RETIRE_BATCH, + }); const counts = { - found: rows.length, - retired: 0, + found: perStatus(), + retired: perStatus(), skippedRunning: 0, skippedChanged: 0, deferred: 0, failed: 0, }; + for (const row of rows) counts.found[row.status]++; for (const [index, row] of rows.entries()) { if (outOfTime()) { counts.deferred = rows.length - index; break; } + const name = `${row.id} (${row.providerSandboxId})`; try { - if (await deps.boxIsRunning(row.providerSandboxId)) { + // With billing off no boot row guards a wake in flight; the listing does. + if ( + row.status === "asleep" && + RUNNING_STATUSES.has(status.get(row.providerSandboxId) ?? "") + ) { counts.skippedRunning++; + decide({ + rule: 1, + kind: "workspace", + name, + action: "keep", + reason: "asleep past its wait, but its box runs a session now", + }); continue; } - if (!(await deps.retireIdleWorkspace(row, cutoff))) { + if (!(await deps.retireWorkspace(row, now))) { counts.skippedChanged++; + decide({ + rule: 1, + kind: "workspace", + name, + action: "keep", + reason: `no longer ${row.status} past its wait: changed since it was read`, + }); continue; } - counts.retired++; - retiredNames.add(row.providerSandboxId); + counts.retired[row.status]++; + retired.add(row.providerSandboxId); + decide({ + rule: 1, + kind: "workspace", + name, + action: "delete", + reason: `${row.status} for more than ${WAIT_MS[row.status] / DAY_MS} days`, + }); await deps.pause(TEARDOWN_SPACING_MS); } catch (error) { counts.failed++; @@ -508,191 +538,233 @@ async function reapIdle({ return counts; } -async function reapArchived({ - now, - deps, - boxes, - outOfTime, -}: { - now: number; - deps: RetentionDeps; - boxes: ListedSandbox[]; - outOfTime: () => boolean; -}): Promise> { - const cutoff = new Date(now - ARCHIVED_BACKSTOP_AFTER_MS); - const found = await deps.countArchivedPastGrace(cutoff); - const lingering = await deps.findArchivedPastGraceAmong( - cutoff, - boxes.map((box) => box.name), - ); - const batch = lingering.slice(0, ARCHIVED_BATCH); +async function pruneGoldens({ deps, boxes, outOfTime, decide, retired }: Ctx) { + const listed = new Set(boxes.map((box) => box.name)); + const envs = await deps.listEnvironmentGoldens(); const counts = { - found, - alreadyDeleted: found - lingering.length, + environments: envs.length, + kept: 0, deleted: 0, - skipped: 0, - deferred: lingering.length - batch.length, + deferred: 0, failed: 0, }; - for (const [index, row] of batch.entries()) { + const due: string[] = []; + for (const env of envs) { + const drop = new Set( + env.archived + ? env.goldens + : goldensToDelete( + env.goldens.map((goldenRef) => ({ goldenRef })), + env.activeRef ?? "", + ).map((golden) => golden.goldenRef), + ); + for (const golden of env.goldens) { + if (drop.has(golden) && listed.has(golden)) { + due.push(golden); + continue; + } + if (drop.has(golden)) continue; + counts.kept++; + decide({ + rule: 2, + kind: "golden", + name: golden, + action: "keep", + reason: + golden === env.activeRef + ? `active golden of environment ${env.id}` + : `among the newest goldens of environment ${env.id}`, + }); + } + } + const shared = await deps.namedByLiveRow(due); + const batch = due.filter((name) => !shared.has(name) && !retired.has(name)); + for (const name of due.filter((name) => shared.has(name))) { + counts.kept++; + decide({ + rule: 2, + kind: "golden", + name, + action: "keep", + reason: "a live workspace or environment still names it", + }); + } + const work = batch.slice(0, GOLDEN_BATCH); + counts.deferred = batch.length - work.length; + for (const [index, name] of work.entries()) { if (outOfTime()) { - counts.deferred += batch.length - index; + counts.deferred += work.length - index; break; } try { - const outcome = row.deletedAt - ? await deps.reapArchived({ - cloudWorkspaceId: row.id, - archivedAt: row.deletedAt.toISOString(), - stage: "delete", - }) - : await deps.reapUndatedArchived({ - cloudWorkspaceId: row.id, - archivedBefore: cutoff, - }); - if (outcome === "reaped") counts.deleted++; - else counts.skipped++; + await deps.tearDownBox(name); + retired.add(name); + counts.deleted++; + decide({ + rule: 2, + kind: "golden", + name, + action: "delete", + reason: "its environment is archived or keeps newer goldens", + }); + await deps.pause(TEARDOWN_SPACING_MS); } catch (error) { counts.failed++; - Sentry.captureException(error, { - tags: { "cloud_workspace.id": row.id }, - }); + Sentry.captureException(error, { extra: { golden: name } }); } } return counts; } -async function reapStranded({ +async function deleteLeftovers(ctx: Ctx) { + return { + boxes: await deleteLeftoverBoxes(ctx), + ...(await deleteLeftoverDrives(ctx)), + }; +} + +async function deleteLeftoverBoxes({ now, deps, boxes, - retiredNames, outOfTime, -}: { - now: number; - deps: RetentionDeps; - boxes: ListedSandbox[]; - retiredNames: Set; - outOfTime: () => boolean; -}): Promise> { - const cutoffs = { - failed: new Date(now - RETENTION_IDLE_MS), - provisioning: new Date(now - STRANDED_PROVISIONING_MS), - }; - const rows = await deps.findStranded({ - failedCutoff: cutoffs.failed, - provisioningCutoff: cutoffs.provisioning, - names: boxes.map((box) => box.name), - limit: STRANDED_BATCH, - }); - const counts = { - found: rows.length, - retired: 0, - skippedChanged: 0, - deferred: 0, - failed: 0, - }; - for (const [index, row] of rows.entries()) { + decide, + retired, +}: Ctx): Promise { + const counts = { kept: 0, deleted: 0, deferred: 0, failed: 0 }; + const fresh = boxes.filter((box) => !retired.has(box.name)); + const named = await deps.namedByAnyRow(fresh.map((box) => box.name)); + const due: string[] = []; + for (const box of fresh) { + const reason = named.has(box.name) + ? "a production row names it" + : now - box.createdAt < LEFTOVER_AGE_MS + ? "made less than 7 days ago" + : null; + if (reason) { + counts.kept++; + decide({ rule: 3, kind: "box", name: box.name, action: "keep", reason }); + } else { + due.push(box.name); + } + } + const work = due.slice(0, LEFTOVER_BOX_BATCH); + counts.deferred = due.length - work.length; + for (const [index, name] of work.entries()) { if (outOfTime()) { - counts.deferred = rows.length - index; + counts.deferred += work.length - index; break; } try { - if (!(await deps.retireStrandedWorkspace(row, cutoffs))) { - counts.skippedChanged++; - continue; - } - counts.retired++; - retiredNames.add(row.providerSandboxId); + await deps.tearDownBox(name); + counts.deleted++; + decide({ + rule: 3, + kind: "box", + name, + action: "delete", + reason: "no production row names it, made more than 7 days ago", + }); await deps.pause(TEARDOWN_SPACING_MS); } catch (error) { counts.failed++; - Sentry.captureException(error, { - tags: { "cloud_workspace.id": row.id }, - }); + Sentry.captureException(error, { extra: { providerSandboxId: name } }); } } return counts; } -function isOldAndSettled(box: ListedSandbox, now: number): boolean { - return ( - SETTLED_STATUSES.has(box.status) && - now - box.createdAt > ORPHAN_MIN_AGE_MS && - now - box.statusUpdatedAt > ORPHAN_MIN_AGE_MS - ); -} - -async function reapOrphans({ +async function deleteLeftoverDrives({ now, deps, boxes, - retiredNames, + drives, outOfTime, -}: { - now: number; - deps: RetentionDeps; - boxes: ListedSandbox[]; - retiredNames: Set; - outOfTime: () => boolean; -}): Promise> { - const ours = boxes.filter( - (box) => box.kind === "workspace" || box.kind === "environment", + decide, +}: Ctx): Promise<{ drives: Counts; agentsDrives: Counts }> { + const mountedBy = new Map(); + for (const box of boxes) { + for (const drive of box.mounts) mountedBy.set(drive, box.name); + } + const current = await deps.currentBundleSha(); + const workspaceDrives = drives.filter( + (drive) => !SANDBOX_AGENT_TOOLS_DRIVE_PATTERN.test(drive.name), ); - const candidates = ours - .filter((box) => isOldAndSettled(box, now) && !retiredNames.has(box.name)) - .map((box) => box.name); - const named = await deps.namedSandboxes(candidates); - const orphaned = candidates.filter((name) => !named.has(name)); - const batch = orphaned.slice(0, ORPHAN_BATCH); - let queued = 0; - let failed = 0; - let unstarted = 0; - for (const [index, name] of batch.entries()) { - if (outOfTime()) { - unstarted = batch.length - index; - break; + const named = await deps.namedDrives(workspaceDrives.map((d) => d.name)); + const keepReason = (drive: ListedDrive): string | null => { + const mounter = mountedBy.get(drive.name); + if (mounter) return `mounted by box ${mounter}`; + if (drive.attachedTo) return `attached to ${drive.attachedTo}`; + if (SANDBOX_AGENT_TOOLS_DRIVE_PATTERN.test(drive.name)) { + if (!current) return "no current release recorded"; + if (drive.name.endsWith(`-${current.slice(0, 12)}`)) { + return "the current release's"; + } + if (now - drive.createdAt < NEW_AGENTS_DRIVE_MS) { + return "published less than a day ago: its release may not be recorded yet"; + } + return null; } + if (named.has(drive.name)) return "a production row names it"; + if (now - drive.createdAt < LEFTOVER_AGE_MS) { + return "made less than 7 days ago"; + } + return null; + }; + const results = { + drives: { kept: 0, deleted: 0, deferred: 0, failed: 0 }, + agentsDrives: { kept: 0, deleted: 0, deferred: 0, failed: 0 }, + }; + let started = 0; + for (const drive of drives) { + const agents = SANDBOX_AGENT_TOOLS_DRIVE_PATTERN.test(drive.name); + const counts = agents ? results.agentsDrives : results.drives; + const kind = agents ? "agents drive" : "drive"; + const reason = keepReason(drive); + if (reason) { + counts.kept++; + decide({ rule: 3, kind, name: drive.name, action: "keep", reason }); + continue; + } + if (started >= DRIVE_BATCH || outOfTime()) { + counts.deferred++; + continue; + } + started++; try { - await deps.tearDownOrphan(name); - queued++; - await deps.pause(TEARDOWN_SPACING_MS); + await deps.deleteDrive(drive.name); + counts.deleted++; + decide({ + rule: 3, + kind, + name: drive.name, + action: "delete", + reason: agents + ? "not the current release's, and no box mounts it" + : "no production row names it and no box mounts it, made more than 7 days ago", + }); } catch (error) { - failed++; - Sentry.captureException(error, { extra: { providerSandboxId: name } }); + counts.failed++; + decide({ + rule: 3, + kind, + name: drive.name, + action: "keep", + reason: `Vercel refused the delete: ${String(error).slice(0, 200)}`, + }); } } - return { - listed: ours.length, - orphaned: orphaned.length, - queued, - deferred: orphaned.length - batch.length + unstarted, - failed, - }; + return results; } -function flatten(run: RetentionRun): Record { - return { - ...Object.fromEntries( - Object.entries(run.idle).map(([key, value]) => [`idle.${key}`, value]), - ), - ...Object.fromEntries( - Object.entries(run.archived).map(([key, value]) => [ - `archived.${key}`, - value, - ]), - ), - ...Object.fromEntries( - Object.entries(run.stranded).map(([key, value]) => [ - `stranded.${key}`, - value, - ]), - ), - ...Object.fromEntries( - Object.entries(run.orphans).map(([key, value]) => [ - `orphans.${key}`, - value, - ]), - ), - }; +function flatten(value: object, prefix = ""): Record { + const flat: Record = {}; + for (const [key, entry] of Object.entries(value)) { + if (entry && typeof entry === "object") { + Object.assign(flat, flatten(entry, `${prefix}${key}.`)); + } else { + flat[`${prefix}${key}`] = entry; + } + } + return flat; } diff --git a/packages/trpc/src/router/cloud-workspace/teardown-guard.ts b/packages/trpc/src/router/cloud-workspace/teardown-guard.ts index 4b50e65b10b..fdd626bb8c1 100644 --- a/packages/trpc/src/router/cloud-workspace/teardown-guard.ts +++ b/packages/trpc/src/router/cloud-workspace/teardown-guard.ts @@ -30,6 +30,27 @@ export async function namedSandboxes( return new Set([...workspaces, ...goldens].map((row) => row.name)); } +/** + * The names among `names` that any production row names: a workspace in any + * status, or an environment, archived ones included (rule 2 deletes those). + */ +export async function namedByAnyRow( + names: readonly string[], +): Promise> { + if (names.length === 0) return new Set(); + const [workspaces, goldens] = await Promise.all([ + db + .select({ name: cloudWorkspaces.providerSandboxId }) + .from(cloudWorkspaces) + .where(inArray(cloudWorkspaces.providerSandboxId, [...names])), + db + .select({ name: environments.sourceRef }) + .from(environments) + .where(inArray(environments.sourceRef, [...names])), + ]); + return new Set([...workspaces, ...goldens].map((row) => row.name)); +} + export async function liveReferences(args: { providerSandboxId: string; }): Promise<{ sandbox: boolean }> { diff --git a/packages/trpc/src/router/cloud-workspace/teardown.test.ts b/packages/trpc/src/router/cloud-workspace/teardown.test.ts index a7e543cd9f3..9d812af0340 100644 --- a/packages/trpc/src/router/cloud-workspace/teardown.test.ts +++ b/packages/trpc/src/router/cloud-workspace/teardown.test.ts @@ -17,6 +17,7 @@ let deleteFails = false; let queued: Array<{ path: string; body: unknown; delaySeconds?: number }> = []; let updateMatches = true; let renamedTo: unknown; +let written: unknown; stub(guard, { liveReferences: async () => ({ sandbox: namedOnRead.shift() ?? false }), @@ -35,6 +36,7 @@ stub(db, { update: () => ({ set: (values: { providerSandboxId: string }) => { renamedTo = values.providerSandboxId; + written = values; return { where: () => ({ returning: async () => (updateMatches ? [{ id: "ws_1" }] : []), @@ -96,13 +98,16 @@ describe("teardownSandbox", () => { describe("retireAndTearDown", () => { const row = { id: "ws_1", providerSandboxId: "ws-old", driveName: null }; - test("gives the row a new box name and queues the old box's teardown only when the update matched", async () => { + test("gives the row a new box name, and its status in the same write, and queues the old box's teardown only when the update matched", async () => { updateMatches = false; expect(await retireAndTearDown(row, undefined)).toBe(false); expect(queued).toEqual([]); updateMatches = true; - expect(await retireAndTearDown(row, undefined)).toBe(true); + expect(await retireAndTearDown(row, undefined, { status: "failed" })).toBe( + true, + ); expect(renamedTo).toMatch(/^ws-ws_1[0-9a-f]{8}$/); + expect(written).toMatchObject({ status: "failed" }); expect(queued).toMatchObject([ { path: "/api/cloud-workspaces/teardown", body: input }, ]); diff --git a/packages/trpc/src/router/cloud-workspace/teardown.ts b/packages/trpc/src/router/cloud-workspace/teardown.ts index 0ce81e9ebbd..48f8ae94586 100644 --- a/packages/trpc/src/router/cloud-workspace/teardown.ts +++ b/packages/trpc/src/router/cloud-workspace/teardown.ts @@ -104,16 +104,18 @@ export async function deleteSandboxAndDriveLater(args: { /** * Takes the row's box name out of use in the same write that commits to * deleting the box, then queues its teardown. A wake after it finds a name - * that does not exist, never the box being deleted. Returns false when the - * row no longer matches `condition` or names another box. + * that does not exist, never the box being deleted. `set` changes the row in + * that same write, such as the status a retired row ends in. Returns false + * when the row no longer matches `condition` or names another box. */ export async function retireAndTearDown( row: { id: string; providerSandboxId: string; driveName: string | null }, condition: SQL | undefined, + set: Partial = {}, ): Promise { const [retired] = await db .update(cloudWorkspaces) - .set({ providerSandboxId: nextSandboxNameFor(row.id) }) + .set({ ...set, providerSandboxId: nextSandboxNameFor(row.id) }) .where( and( eq(cloudWorkspaces.id, row.id), diff --git a/packages/trpc/src/router/environment/environment.test.ts b/packages/trpc/src/router/environment/environment.test.ts index 76404214510..64deffa784b 100644 --- a/packages/trpc/src/router/environment/environment.test.ts +++ b/packages/trpc/src/router/environment/environment.test.ts @@ -58,7 +58,13 @@ stub(access, { branch: "main", }), }); -stub(sandbox, { workspaceRepositories: async () => [] }); +let deletedBoxes: string[] = []; +stub(sandbox, { + workspaceRepositories: async () => [], + deleteSandbox: async (name: string) => { + deletedBoxes.push(name); + }, +}); stub(plan, { resolveMachineSize: async (_: string, name: string | null | undefined) => name === "large" || name == null @@ -95,6 +101,22 @@ beforeEach(() => { }; isAdmin = false; written = []; + deletedBoxes = []; +}); + +describe("environment.archive", () => { + test("only sets a status: the cleanup job deletes its golden (rule 2)", async () => { + environment = { + ...environment, + sourceKind: "fork", + sourceRef: "env-golden", + }; + expect(await caller.environment.archive({ id: ENVIRONMENT_ID })).toEqual({ + archived: true, + }); + expect(written).toEqual([{ archivedAt: expect.any(Date) }]); + expect(deletedBoxes).toEqual([]); + }); }); describe("environment.update machineSize", () => { diff --git a/packages/trpc/src/router/environment/environment.ts b/packages/trpc/src/router/environment/environment.ts index 060826df00e..4982b661507 100644 --- a/packages/trpc/src/router/environment/environment.ts +++ b/packages/trpc/src/router/environment/environment.ts @@ -722,19 +722,11 @@ export const environmentRouter = { .mutation(async ({ ctx, input }) => { const environment = await loadEnvironment(input.id, ctx); await assertCloudAccess(ctx, environment.organizationId); - const [archived] = await db + await db .update(environments) .set({ archivedAt: new Date() }) - .where(eq(environments.id, environment.id)) - .returning({ - sourceKind: environments.sourceKind, - sourceRef: environments.sourceRef, - }); - // A golden is one environment's alone, and an archived environment - // never forks from it again; without this it bills storage forever. - if (archived?.sourceKind === "fork") { - await deleteSandbox(archived.sourceRef); - } + .where(eq(environments.id, environment.id)); + // Only a status: the cleanup job deletes an archived environment's goldens (rule 2). return { archived: true }; }), } satisfies TRPCRouterRecord; diff --git a/plans/done/20261118-1500-cloud-workspace-retention-reaper.md b/plans/done/20261118-1500-cloud-workspace-retention-reaper.md index f3a2e7cb8fb..cd807bb9142 100644 --- a/plans/done/20261118-1500-cloud-workspace-retention-reaper.md +++ b/plans/done/20261118-1500-cloud-workspace-retention-reaper.md @@ -90,6 +90,9 @@ None blocking. The QStash schedule must be created in the Upstash console by som - Decision: a stranded stage (production only, after the archived stage, batch 50) for the drive-liveness property that #9914's model found failing. It selects rows `provisioning` for `STRANDED_PROVISIONING_MS` (1 day, by `created_at`, `updated_at` and no boot since), and rows `failed` for `RETENTION_IDLE_MS` (by `coalesce(failed_at, updated_at)`) whose box is in this run's listing. It retires each through `retireAndTearDown` with the same status-and-age predicate as the condition; a stranded provisioning row then fails as `create_failed`. On main this covers boxes only; the drive half needs #9847's `drive_name` (see the rebase list in the PR). Rationale: a stranded provisioning row leaves the selection by its status, a failed one because its retired name is in no listing, so no row can fill an oldest-first batch forever. Failed rows wait 30 days, because a failed row keeps its box for diagnosis. A call that does not complete is retried by a later run: a stranded row whose fail threw is selected again a day later (the rename moved `updated_at`); a box whose teardown was not queued is an orphan. On 2026-11-19 production had 2 failed rows (1 failed over 30 days ago) and no stranded ones. Date/Author: 2026-11-19, orchestrator. +- Decision: rebuild #9987 to the agreed end state (page 20261119-sandbox-lifecycle-end-state, model #10006). The cleanup job applies three rules from one table and is the only thing that deletes for age: rule 1 (one select over the wait table: asleep 30 d since last use, archived 7 d, failed 30 d, stuck creating 1 d with no boot; each through one conditional retire write, then the retried teardown), rule 2 (archived environments' goldens; live environments through `goldensToDelete`, the same as #10000's), rule 3 (unnamed boxes, drives and goldens older than 7 days; agents drives that are not current and no box mounts). It runs only in production and deletes nothing unless the whole box and drive listing was read (#9943's checks). `deleteSandbox` and `deleteDrive` delete only when `VERCEL_ENV` is production; staging deploys with `--prod` on the production database, so it counts. Archive, a failed create, the provision race and environment archive only set a status; archive's delayed delete message is gone, its 60 s stop stays, and an unarchive after 7 days is refused. The release no longer prunes agents drives. The stranded and archived-backstop stages are dropped: rule 1 covers them; the orphan stage became rule 3. + Rationale: one deleter with one rule set, checked by a model, replaces five places that deleted on their own schedule. Stricter than the model in four places, all keeps: an asleep row whose box runs now; a workspace drive a box mounts; an agents drive younger than a day (a release publishes drives before it records its bundle); nothing at all on an incomplete listing. Date/Author: 2026-11-19, orchestrator. + ## Outcomes & Retrospective All seven parts of the brief are done except the drives. Workspace drives do not exist on main, so the drive half is a seam: `TeardownSandboxInput.driveName` is typed `null`, and #9847's version of `teardownSandbox` replaces this one at the same route with the same payload. The reaper's measure of last activity is deliberately later than Vercel's: the 412 rows whose `sandbox_expires_at` was moved forward on 2026-11-16 are reaped no earlier than 2026-12-16, and until then Vercel's own expiry and the 410 path still apply to them. @@ -97,3 +100,5 @@ All seven parts of the brief are done except the drives. Workspace drives do not Revision note (2026-11-18): aligned with #9847's `retireAndTearDown` and `deleteSandboxAndDriveLater`; the idle stage's single update became two writes, and teardowns are paced in the loop instead of by `delaySeconds`. Revision note (2026-11-19): added the stranded stage for failed and stuck-provisioning rows, for the drive-liveness property in #9914's model. + +Revision note (2026-11-19): rebuilt to the end-state rules table (rules 1 to 3) with the production-only delete check and status-only archive, failed create and environment archive. From 094a652bac381d878c404f3196a9eaf43a6f2ba8 Mon Sep 17 00:00:00 2001 From: Satya Patel <14907857+saddlepaddle@users.noreply.github.com> Date: Thu, 19 Nov 2026 06:16:30 +0000 Subject: [PATCH 7/8] refactor(sandbox): one production-only delete check, #9847's sandboxDeletesAllowed Same file and signature as #9847 (038f603b): teardownSandbox returns skipped outside production before it stops anything, and deleteDrive and a replaced golden's delete check it too. deleteSandbox itself stays unchecked so the release probe and real-sandbox still delete their own temporary boxes. The test preload sets VERCEL_ENV=production. --- docs/cloud-sandbox-mismatches.md | 7 +++++-- packages/trpc/src/lib/sandbox/deletes.ts | 11 +++++++++++ packages/trpc/src/lib/sandbox/index.ts | 2 +- packages/trpc/src/lib/sandbox/vercel.test.ts | 19 ++++++------------- packages/trpc/src/lib/sandbox/vercel.ts | 19 ++----------------- .../router/cloud-workspace/teardown.test.ts | 10 ++++++++++ .../src/router/cloud-workspace/teardown.ts | 16 ++++++++++++++-- .../src/router/environment/environment.ts | 6 +++++- packages/trpc/test/setup-env.ts | 2 ++ 9 files changed, 56 insertions(+), 36 deletions(-) create mode 100644 packages/trpc/src/lib/sandbox/deletes.ts diff --git a/docs/cloud-sandbox-mismatches.md b/docs/cloud-sandbox-mismatches.md index 21bd2b68b1e..52e8d922dd3 100644 --- a/docs/cloud-sandbox-mismatches.md +++ b/docs/cloud-sandbox-mismatches.md @@ -805,8 +805,11 @@ only thing that deletes for age. It applies three rules from one table: stopped ones included, mounts it. Everything else only sets a status: archive (its 60 s stop stays), a failed -create and a 410 on wake. Only production deletes: `deleteSandbox` and -`deleteDrive` log "would delete" unless `VERCEL_ENV` is `production`, because +create and a 410 on wake. Only production deletes: `sandboxDeletesAllowed` +(`lib/sandbox/deletes.ts`), checked by the teardown job, the drive delete and a +replaced golden's delete, logs "would delete" unless `VERCEL_ENV` is +`production`. `deleteSandbox` itself stays unchecked so a job still deletes the +temporary boxes it made (the release probe, real-sandbox). This is because every deployment shares one Vercel project and a preview database copied from production names production's boxes. Staging deploys as production on the production database, so it counts. The job reads the whole box and drive diff --git a/packages/trpc/src/lib/sandbox/deletes.ts b/packages/trpc/src/lib/sandbox/deletes.ts new file mode 100644 index 00000000000..43ff545dfdd --- /dev/null +++ b/packages/trpc/src/lib/sandbox/deletes.ts @@ -0,0 +1,11 @@ +/** Previews and local dev use production's Vercel project with rows copied from its database: only production deletes. */ +export function sandboxDeletesAllowed( + what: string, + vercelEnv: string | undefined = process.env.VERCEL_ENV, +): boolean { + if (vercelEnv === "production") return true; + console.log( + `[sandbox] would delete ${what}; VERCEL_ENV is ${vercelEnv ?? "unset"}, not production`, + ); + return false; +} diff --git a/packages/trpc/src/lib/sandbox/index.ts b/packages/trpc/src/lib/sandbox/index.ts index 993e5f82437..45808addf3b 100644 --- a/packages/trpc/src/lib/sandbox/index.ts +++ b/packages/trpc/src/lib/sandbox/index.ts @@ -26,6 +26,7 @@ export { } from "./box-pressure"; export { buildSandboxClaim } from "./claim"; export { deriveSandboxCredentials } from "./credentials"; +export { sandboxDeletesAllowed } from "./deletes"; export { listRemoteBranches, type RemoteBranch, @@ -55,7 +56,6 @@ export { export { type ActiveWorkspaceSandbox, type BoxSession, - canDelete, deleteDrive, deleteSandbox, describeSandbox, diff --git a/packages/trpc/src/lib/sandbox/vercel.test.ts b/packages/trpc/src/lib/sandbox/vercel.test.ts index f455966eb3c..e8bd65409f9 100644 --- a/packages/trpc/src/lib/sandbox/vercel.test.ts +++ b/packages/trpc/src/lib/sandbox/vercel.test.ts @@ -8,10 +8,10 @@ import { } from "bun:test"; import { APIError, Drive, Sandbox } from "@vercel/sandbox"; import { stub } from "../../../test/stub"; +import { sandboxDeletesAllowed } from "./deletes"; import { agentToolsMounts, applySandboxPolicy, - canDelete, deleteDrive, deleteSandbox, describeSandbox, @@ -312,29 +312,22 @@ describe("a caller that is not a wake leaves a stopped box stopped", () => { }); describe("only production deletes", () => { - stub(Sandbox, { - get: async () => { - throw new Error("reached Vercel"); - }, - }); stub(Drive, { get: async () => { throw new Error("reached Vercel"); }, }); - test("outside production a box or drive delete logs and returns without reaching Vercel", async () => { - const vercelEnv = process.env.VERCEL_ENV; + test("outside production a drive delete logs and returns without reaching Vercel", async () => { process.env.VERCEL_ENV = "preview"; try { - await deleteSandbox("ws-box"); await deleteDrive("drive-1"); } finally { - process.env.VERCEL_ENV = vercelEnv; + process.env.VERCEL_ENV = "production"; } - expect(canDelete("production")).toBe(true); - expect(canDelete("preview")).toBe(false); - expect(canDelete(undefined)).toBe(false); + expect(sandboxDeletesAllowed("box b", "production")).toBe(true); + expect(sandboxDeletesAllowed("box b", "preview")).toBe(false); + expect(sandboxDeletesAllowed("box b", "development")).toBe(false); }); }); diff --git a/packages/trpc/src/lib/sandbox/vercel.ts b/packages/trpc/src/lib/sandbox/vercel.ts index f062aa5fb05..4c804707672 100644 --- a/packages/trpc/src/lib/sandbox/vercel.ts +++ b/packages/trpc/src/lib/sandbox/vercel.ts @@ -34,6 +34,7 @@ import { CODEX_SYSTEM_CONFIG_PATH, type SandboxFile, } from "./credentials"; +import { sandboxDeletesAllowed } from "./deletes"; export const HOST_SERVICE_PORT = SANDBOX_PORTS.hostService; /** @@ -1101,24 +1102,9 @@ export function listProjectDrives(): Promise { }); } -/** - * Only production deletes. Staging deploys as production on the same - * database, so it counts; a preview or local database, copied from - * production, names production's boxes and only logs. - */ -export function canDelete(vercelEnv = process.env.VERCEL_ENV): boolean { - return vercelEnv === "production"; -} - -function wouldDelete(kind: string, name: string): boolean { - if (canDelete()) return false; - console.log(`[sandbox] would delete ${kind} ${name}: not production`); - return true; -} - /** Throws while a box holds it; a drive already gone is the state we wanted. */ export async function deleteDrive(name: string): Promise { - if (wouldDelete("drive", name)) return; + if (!sandboxDeletesAllowed(`drive ${name}`)) return; const drive = await Drive.get({ ...credentials(), name }).catch((error) => { if (isNotFound(error)) return null; throw error; @@ -1128,7 +1114,6 @@ export async function deleteDrive(name: string): Promise { /** Best-effort: a sandbox already gone is the state we wanted. */ export async function deleteSandbox(providerSandboxId: string): Promise { - if (wouldDelete("sandbox", providerSandboxId)) return; const sandbox = await getSandbox(providerSandboxId); if (!sandbox) return; // Snapshots outlive a sandbox by default and keep billing storage. diff --git a/packages/trpc/src/router/cloud-workspace/teardown.test.ts b/packages/trpc/src/router/cloud-workspace/teardown.test.ts index 9d812af0340..09051b05334 100644 --- a/packages/trpc/src/router/cloud-workspace/teardown.test.ts +++ b/packages/trpc/src/router/cloud-workspace/teardown.test.ts @@ -67,6 +67,16 @@ beforeEach(() => { const input = { providerSandboxId: "ws-old", driveName: null, attempt: 0 }; describe("teardownSandbox", () => { + test("outside production it stops and deletes nothing", async () => { + process.env.VERCEL_ENV = "preview"; + try { + expect(await teardownSandbox(input)).toBe("skipped"); + } finally { + process.env.VERCEL_ENV = "production"; + } + expect(calls).toEqual([]); + }); + test("stops, then deletes a box no row names", async () => { expect(await teardownSandbox(input)).toBe("deleted"); expect(calls).toEqual(["stop:ws-old", "delete:ws-old"]); diff --git a/packages/trpc/src/router/cloud-workspace/teardown.ts b/packages/trpc/src/router/cloud-workspace/teardown.ts index 48f8ae94586..3ba7077351f 100644 --- a/packages/trpc/src/router/cloud-workspace/teardown.ts +++ b/packages/trpc/src/router/cloud-workspace/teardown.ts @@ -2,7 +2,11 @@ import * as Sentry from "@sentry/core"; import { db } from "@superset/db/client"; import { cloudWorkspaces } from "@superset/db/schema"; import { and, eq, type SQL } from "drizzle-orm"; -import { deleteSandbox, stopSandbox } from "../../lib/sandbox"; +import { + deleteSandbox, + sandboxDeletesAllowed, + stopSandbox, +} from "../../lib/sandbox"; import { publishCloudWorkspaceJob } from "./jobs"; import { nextSandboxNameFor } from "./provision"; import { liveReferences } from "./teardown-guard"; @@ -17,7 +21,12 @@ export interface TeardownSandboxInput { export const TEARDOWN_RETRY_SECONDS = 30; export const TEARDOWN_ATTEMPTS = 20; -export type TeardownOutcome = "deleted" | "kept" | "retrying" | "abandoned"; +export type TeardownOutcome = + | "deleted" + | "kept" + | "retrying" + | "abandoned" + | "skipped"; /** * Stops the box, then deletes it with its snapshots. Before each destructive @@ -27,6 +36,9 @@ export type TeardownOutcome = "deleted" | "kept" | "retrying" | "abandoned"; export async function teardownSandbox( input: TeardownSandboxInput, ): Promise { + if (!sandboxDeletesAllowed(`box ${input.providerSandboxId} and no drive`)) { + return "skipped"; + } try { return await teardownOnce(input); } catch (error) { diff --git a/packages/trpc/src/router/environment/environment.ts b/packages/trpc/src/router/environment/environment.ts index 4982b661507..280e9779114 100644 --- a/packages/trpc/src/router/environment/environment.ts +++ b/packages/trpc/src/router/environment/environment.ts @@ -27,6 +27,7 @@ import { primaryRepository, promoteSandboxToEnvironment, RepositoryError, + sandboxDeletesAllowed, sortRepositories, workspaceRepositories, } from "../../lib/sandbox"; @@ -625,7 +626,10 @@ export const environmentRouter = { await restart(); throw error; }); - if (target?.sourceKind === "fork") { + if ( + target?.sourceKind === "fork" && + sandboxDeletesAllowed(`golden ${target.sourceRef}`) + ) { await deleteSandbox(target.sourceRef).catch((error: unknown) => console.error( `[environment/promote] ${target.id} replaced; could not delete its previous golden ${target.sourceRef}`, diff --git a/packages/trpc/test/setup-env.ts b/packages/trpc/test/setup-env.ts index 9d1b00eb4c0..246920a0b1c 100644 --- a/packages/trpc/test/setup-env.ts +++ b/packages/trpc/test/setup-env.ts @@ -2,6 +2,8 @@ import "../../../scripts/test-preload.ts"; process.env.SKIP_ENV_VALIDATION = "1"; process.env.BILLING_PROVIDER = "stripe"; +// `sandboxDeletesAllowed` deletes nothing outside production; the tests check the deletes. +process.env.VERCEL_ENV = "production"; // Importing a module that constructs a client throws without these. None // connects until it is used. process.env.STRIPE_SECRET_KEY ??= "sk_test_placeholder"; From 4012fd8627547d6a30fe5caffd17cc61647cb0a3 Mon Sep 17 00:00:00 2001 From: Satya Patel <14907857+saddlepaddle@users.noreply.github.com> Date: Thu, 19 Nov 2026 06:30:27 +0000 Subject: [PATCH 8/8] feat(cloud-workspace): a dry run of the cleanup job reapCloudWorkspaceRetention takes dryRun: it reads everything, listing checks included, and replaces every writing or deleting dependency (MUTATING_DEPS) with one that only records, logging "would delete". scripts/cleanup-dry-run.ts runs it against a real database and the real Vercel listing: it refuses any role but readonly, sets VERCEL_ENV to development, and makes the SDK's stop, update, create and delete throw. --- packages/trpc/scripts/cleanup-dry-run.ts | 289 ++++++++++++++++++ packages/trpc/src/lib/sandbox/vercel.ts | 2 + .../router/cloud-workspace/retention.test.ts | 36 +++ .../src/router/cloud-workspace/retention.ts | 39 ++- 4 files changed, 362 insertions(+), 4 deletions(-) create mode 100644 packages/trpc/scripts/cleanup-dry-run.ts diff --git a/packages/trpc/scripts/cleanup-dry-run.ts b/packages/trpc/scripts/cleanup-dry-run.ts new file mode 100644 index 00000000000..ab913415981 --- /dev/null +++ b/packages/trpc/scripts/cleanup-dry-run.ts @@ -0,0 +1,289 @@ +/** + * What the cleanup job's next run would do, against a real database and the + * real Vercel listing, with nothing written or deleted. Runs the job itself + * (listing checks included) with `dryRun`, then reports each rule's + * decisions grouped by kind, action and reason. + * + * Four guards keep it from deleting anything: + * 1. it refuses unless DATABASE_URL logs in as the `readonly` role, so no row can change; + * 2. VERCEL_ENV is set to "development" before anything loads, so `sandboxDeletesAllowed` refuses; + * 3. every SDK call that stops, updates, creates or deletes a box or drive throws; + * 4. the job runs with `dryRun`, which replaces every writing or deleting dependency. + * + * Usage: DATABASE_URL= VERCEL_SANDBOX_TOKEN=… VERCEL_SANDBOX_TEAM_ID=… \ + * VERCEL_SANDBOX_PROJECT_ID=… bun run packages/trpc/scripts/cleanup-dry-run.ts [--json out.json] + */ + +const databaseUrl = process.env.DATABASE_URL ?? ""; +let role = ""; +try { + role = decodeURIComponent(new URL(databaseUrl).username); +} catch {} +if (role !== "readonly") { + console.error( + `DATABASE_URL must log in as the readonly role, not "${role || "none"}"`, + ); + process.exit(64); +} +for (const key of [ + "VERCEL_SANDBOX_TOKEN", + "VERCEL_SANDBOX_TEAM_ID", + "VERCEL_SANDBOX_PROJECT_ID", +]) { + if (!process.env[key]) { + console.error(`${key} is required`); + process.exit(64); + } +} +process.env.VERCEL_ENV = "development"; +process.env.SKIP_ENV_VALIDATION = "1"; +process.env.DATABASE_URL_UNPOOLED = databaseUrl; + +const { Drive, Sandbox } = await import("@vercel/sandbox"); +const trap = (what: string) => () => { + throw new Error(`dry run reached ${what}`); +}; +const trapped: Array<[object, string]> = [ + [Sandbox.prototype, "delete"], + [Sandbox.prototype, "stop"], + [Sandbox.prototype, "update"], + [Sandbox, "create"], + [Drive.prototype, "delete"], + [Drive, "getOrCreate"], +]; +for (const [target, key] of trapped) { + Object.defineProperty(target, key, { + value: trap(key), + configurable: false, + writable: false, + }); +} + +const { db } = await import("@superset/db/client"); +const { cloudWorkspaces, environments, organizations } = await import( + "@superset/db/schema" +); +const { count, inArray } = await import("drizzle-orm"); +const retention = await import("../src/router/cloud-workspace/retention"); +type Deps = import("../src/router/cloud-workspace/retention").RetentionDeps; +type ListedSandbox = import("../src/lib/sandbox").ListedSandbox; +type ListedDrive = import("../src/lib/sandbox").ListedDrive; + +const now = Date.now(); +let boxes: ListedSandbox[] = []; +let drives: ListedDrive[] = []; +const rows = new Map(); +const lines: string[] = []; + +const base = retention.defaultRetentionDeps; +const deps: Deps = { + ...base, + listSandboxes: async (expected) => { + boxes = await base.listSandboxes(expected); + return boxes; + }, + listDrives: async () => { + drives = await base.listDrives(); + return drives; + }, + findPastWait: async (args) => { + const found = await base.findPastWait(args); + for (const row of found) rows.set(row.id, row); + return found; + }, + log: (line) => lines.push(line), +}; + +const result = await retention.reapCloudWorkspaceRetention({ + now, + production: true, + deps, + dryRun: true, +}); + +type Decision = { + rule: number; + kind: string; + name: string; + action: string; + reason: string; +}; +const decisions: Decision[] = lines + .filter((line) => line.startsWith("[cleanup] {")) + .map((line) => JSON.parse(line.slice("[cleanup] ".length))); +const reached = decisions.filter((d) => d.action === "delete"); +if (reached.length) throw new Error("dry run logged a real delete"); + +const boxByName = new Map(boxes.map((box) => [box.name, box])); +const driveByName = new Map(drives.map((drive) => [drive.name, drive])); +const createdAt = (d: Decision): number | null => { + if (d.kind === "workspace") { + const id = d.name.split(" ")[0] ?? ""; + return rows.get(id)?.createdAt.getTime() ?? null; + } + if (d.kind === "drive" || d.kind === "agents drive") { + return driveByName.get(d.name)?.createdAt ?? null; + } + return boxByName.get(d.name)?.createdAt ?? null; +}; +const iso = (ms: number | null) => (ms ? new Date(ms).toISOString() : "?"); + +// Rule 1's select is capped at a batch; count everything past its wait too. +const pastWait = await db + .select({ status: cloudWorkspaces.status, n: count() }) + .from(cloudWorkspaces) + .where( + retention.waitOverWhere({ now, listedNames: boxes.map((b) => b.name) }), + ) + .groupBy(cloudWorkspaces.status); + +// Where an unnamed box came from: a production row, or a database that is not production. +const unnamed = decisions.filter( + (d) => d.rule === 3 && d.kind === "box" && d.action === "would delete", +); +const tagged = (key: string) => [ + ...new Set( + unnamed.flatMap((d) => { + const value = boxByName.get(d.name)?.tags[key]; + return value ? [value] : []; + }), + ), +]; +const knownWorkspaces = new Set(); +for (let i = 0; i < tagged("workspace").length; i += 500) { + for (const row of await db + .select({ id: cloudWorkspaces.id }) + .from(cloudWorkspaces) + .where( + inArray(cloudWorkspaces.id, tagged("workspace").slice(i, i + 500)), + )) { + knownWorkspaces.add(row.id); + } +} +const knownOrgs = new Set(); +for (let i = 0; i < tagged("org").length; i += 500) { + for (const row of await db + .select({ id: organizations.id }) + .from(organizations) + .where(inArray(organizations.id, tagged("org").slice(i, i + 500)))) { + knownOrgs.add(row.id); + } +} +const originOf = (name: string): string => { + const tags = boxByName.get(name)?.tags ?? {}; + if (!tags.kind) return "untagged (made before tags, or not by the API)"; + if (tags.workspace && knownWorkspaces.has(tags.workspace)) { + return "its production workspace row names another box now"; + } + if (tags.org && !knownOrgs.has(tags.org)) { + return "organization not in production: a preview's or dev's, or deleted"; + } + if (tags.workspace) { + return "workspace not in production: made by a preview or dev database, or its row was deleted"; + } + return `kind ${tags.kind}, no production row`; +}; + +const envRows = await db + .select({ + id: environments.id, + name: environments.name, + sourceRef: environments.sourceRef, + archivedAt: environments.archivedAt, + }) + .from(environments); +const envByRef = new Map(envRows.map((env) => [env.sourceRef, env])); + +const groups = new Map(); +for (const d of decisions) { + const key = `${d.rule}|${d.kind}|${d.action}|${d.reason}`; + groups.set(key, [...(groups.get(key) ?? []), d]); +} +const report = [...groups.entries()] + .map(([key, items]) => { + const [rule = "", kind = "", action = "", reason = ""] = key.split("|"); + const sorted = [...items].sort( + (a, b) => (createdAt(a) ?? Infinity) - (createdAt(b) ?? Infinity), + ); + const oldest = sorted[0]; + return { + rule: Number(rule), + kind, + action, + reason, + count: items.length, + examples: items.slice(0, 5).map((d) => d.name), + oldest: oldest + ? { name: oldest.name, createdAt: iso(createdAt(oldest)) } + : null, + }; + }) + .sort( + (a, b) => + a.rule - b.rule || + a.kind.localeCompare(b.kind) || + a.action.localeCompare(b.action) || + b.count - a.count, + ); + +const wouldDelete = decisions.filter((d) => d.action === "would delete"); +const running = new Set(["pending", "running"]); +const flags = { + runningNowAmongDeletes: wouldDelete + .filter((d) => { + const name = + d.kind === "workspace" ? d.name.match(/\((.*)\)$/)?.[1] : d.name; + return name && running.has(boxByName.get(name)?.status ?? ""); + }) + .map((d) => `${d.rule}:${d.kind}:${d.name}`), + runningNowListed: boxes.filter((box) => running.has(box.status)).length, + environmentGoldensAmongDeletes: wouldDelete + .filter((d) => envByRef.has(d.name)) + .map((d) => { + const env = envByRef.get(d.name); + return `${d.name} (${env?.name}${env?.archivedAt ? ", archived" : ""})`; + }), + internalGoldens: envRows + .filter((env) => /superset/i.test(env.name)) + .map((env) => ({ + environment: env.name, + golden: env.sourceRef, + decision: + decisions.find((d) => d.name === env.sourceRef)?.action ?? "not listed", + })), + workspaceDrivesLike9847: drives + .filter((d) => /^(env-)?drive-/.test(d.name)) + .map((d) => ({ + name: d.name, + createdAt: iso(d.createdAt), + decision: decisions.find((x) => x.name === d.name)?.action ?? "none", + })), + unnamedBoxOrigins: Object.entries( + unnamed.reduce>((acc, d) => { + const origin = originOf(d.name); + acc[origin] = (acc[origin] ?? 0) + 1; + return acc; + }, {}), + ), +}; + +const summary = { + ranAt: new Date(now).toISOString(), + listed: { boxes: boxes.length, drives: drives.length }, + result, + rule1PastWaitInTotal: Object.fromEntries( + pastWait.map((row) => [row.status, row.n]), + ), + report, + flags, +}; +const jsonAt = process.argv.indexOf("--json"); +if (jsonAt > 0 && process.argv[jsonAt + 1]) { + const { writeFile } = await import("node:fs/promises"); + await writeFile( + process.argv[jsonAt + 1] as string, + JSON.stringify(summary, null, 2), + ); +} +console.log(JSON.stringify(summary, null, 2)); +process.exit(0); diff --git a/packages/trpc/src/lib/sandbox/vercel.ts b/packages/trpc/src/lib/sandbox/vercel.ts index 4c804707672..1313143fb46 100644 --- a/packages/trpc/src/lib/sandbox/vercel.ts +++ b/packages/trpc/src/lib/sandbox/vercel.ts @@ -996,6 +996,7 @@ export interface ListedSandbox { status: string; /** What it mounts, running or stopped: a stopped box mounts it again on its next boot. */ mounts: string[]; + tags: Record; } export interface ListedDrive { @@ -1073,6 +1074,7 @@ export function listProjectSandboxes( mounts: Object.values(sandbox.mounts ?? {}).map( (mount) => mount.drive, ), + tags: sandbox.tags ?? {}, })), next: page.pagination.next, }; diff --git a/packages/trpc/src/router/cloud-workspace/retention.test.ts b/packages/trpc/src/router/cloud-workspace/retention.test.ts index 25b5883f0f5..92c0770554d 100644 --- a/packages/trpc/src/router/cloud-workspace/retention.test.ts +++ b/packages/trpc/src/router/cloud-workspace/retention.test.ts @@ -7,6 +7,7 @@ import * as reap from "./reap"; import { type EnvironmentGoldens, LEFTOVER_AGE_MS, + MUTATING_DEPS, RETIRE_BATCH, type RetentionDeps, RUN_BUDGET_MS, @@ -30,6 +31,7 @@ const row = (n: number, status: WaitStatus = "asleep"): WaitingWorkspace => ({ organizationId: "org_1", providerSandboxId: `ws-box${n}`, status, + createdAt: new Date(NOW - 90 * DAY), }); const box = ( @@ -41,6 +43,7 @@ const box = ( createdAt: NOW - 60 * DAY, status: "stopped", mounts: [], + tags: {}, ...overrides, }); @@ -100,6 +103,39 @@ const run = (deps: RetentionDeps, clock?: () => number) => reapCloudWorkspaceRetention({ now: NOW, production: true, deps, clock }); describe("the cleanup job", () => { + test("a dry run reads everything, reaches no write or delete, and logs what it would delete", async () => { + const reached = () => { + throw new Error("a dry run reached a write or a delete"); + }; + const { deps, decisions } = fakeDeps({ + listSandboxes: async () => [box("ws-orphan"), box("env-old")], + namedByAnyRow: async () => new Set(), + findPastWait: async () => [row(1)], + listDrives: async () => [drive("drive-orphan")], + listEnvironmentGoldens: async () => [ + { id: "e", archived: true, activeRef: "env-old", goldens: ["env-old"] }, + ], + }); + const trapped = { + ...deps, + ...Object.fromEntries(MUTATING_DEPS.map((key) => [key, reached])), + } as RetentionDeps; + const result = await reapCloudWorkspaceRetention({ + now: NOW, + production: true, + deps: trapped, + dryRun: true, + }); + expect(result).toMatchObject({ + rule1: { failed: 0 }, + rule2: { deleted: 1, failed: 0 }, + rule3: { boxes: { deleted: 1, failed: 0 }, drives: { deleted: 1 } }, + }); + const actions = new Set(decisions().map((d) => d.action)); + expect(actions.has("delete")).toBe(false); + expect(actions.has("would delete")).toBe(true); + }); + test("runs only in production", async () => { let listed = false; const { deps } = fakeDeps({ diff --git a/packages/trpc/src/router/cloud-workspace/retention.ts b/packages/trpc/src/router/cloud-workspace/retention.ts index db089842384..92511252541 100644 --- a/packages/trpc/src/router/cloud-workspace/retention.ts +++ b/packages/trpc/src/router/cloud-workspace/retention.ts @@ -182,6 +182,7 @@ export interface WaitingWorkspace { organizationId: string; providerSandboxId: string; status: WaitStatus; + createdAt: Date; } async function findPastWait(args: { @@ -195,6 +196,7 @@ async function findPastWait(args: { organizationId: cloudWorkspaces.organizationId, providerSandboxId: cloudWorkspaces.providerSandboxId, status: cloudWorkspaces.status, + createdAt: cloudWorkspaces.createdAt, }) .from(cloudWorkspaces) .where(waitOverWhere(args)) @@ -333,7 +335,7 @@ export interface RetentionDeps { log: (line: string) => void; } -const defaultDeps: RetentionDeps = { +export const defaultRetentionDeps: RetentionDeps = { boxesRunningNow, listSandboxes: listProjectSandboxes, listDrives: listProjectDrives, @@ -351,6 +353,25 @@ const defaultDeps: RetentionDeps = { log: (line) => console.log(line), }; +/** Every dependency that writes a row or deletes in Vercel; a dry run replaces each one. */ +export const MUTATING_DEPS = [ + "retireWorkspace", + "tearDownBox", + "deleteDrive", + "pause", +] as const satisfies readonly (keyof RetentionDeps)[]; + +/** The job's reads, unchanged; its writes and deletes only recorded as if they had worked. */ +export function dryRunDeps(deps: RetentionDeps): RetentionDeps { + const replaced: Pick = { + retireWorkspace: async () => true, + tearDownBox: async () => {}, + deleteDrive: async () => {}, + pause: async () => {}, + }; + return { ...deps, ...replaced }; +} + type Counts = { kept: number; deleted: number; @@ -385,7 +406,7 @@ type Decision = { rule: 1 | 2 | 3; kind: "workspace" | "box" | "golden" | "drive" | "agents drive"; name: string; - action: "keep" | "delete"; + action: "keep" | "delete" | "would delete"; reason: string; }; @@ -399,14 +420,18 @@ type Decision = { export async function reapCloudWorkspaceRetention({ now = Date.now(), production, - deps = defaultDeps, + deps: given = defaultRetentionDeps, clock = Date.now, + dryRun = false, }: { now?: number; production: boolean; deps?: RetentionDeps; clock?: () => number; + /** Reads everything, writes and deletes nothing, and logs "would delete". */ + dryRun?: boolean; }): Promise { + const deps = dryRun ? dryRunDeps(given) : given; if (!production) { deps.log("[cleanup] skipped: only production runs the cleanup job"); return { skipped: "not production" }; @@ -414,7 +439,13 @@ export async function reapCloudWorkspaceRetention({ const startedAt = clock(); const outOfTime = () => clock() - startedAt > RUN_BUDGET_MS; const decide = (decision: Decision) => - deps.log(`[cleanup] ${JSON.stringify(decision)}`); + deps.log( + `[cleanup] ${JSON.stringify( + dryRun && decision.action === "delete" + ? { ...decision, action: "would delete" } + : decision, + )}`, + ); let boxes: ListedSandbox[]; let drives: ListedDrive[]; try {