Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/workflows/publish-cloud-workspace-job.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ on:
options:
- reconcile-usage
- renew-chatgpt-sign-ins
- reap-retention
default: reconcile-usage

jobs:
Expand Down
31 changes: 31 additions & 0 deletions apps/api/src/app/api/cloud-workspaces/jobs/reap-retention/route.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
import * as Sentry from "@sentry/nextjs";
import {
RETENTION_JOB_PATH,
reapCloudWorkspaceRetention,
} from "@superset/trpc/cloud-workspace-retention";
import { verifyQstashRequest } from "@/lib/verifyQstash";

export const dynamic = "force-dynamic";
export const maxDuration = 300;

/**
* Run by a QStash schedule every 6 hours. That schedule lives in the
* Upstash console, not in this repo.
*/
export async function POST(request: Request): Promise<Response> {
const body = await request.text();
const rejected = await verifyQstashRequest(request, body, RETENTION_JOB_PATH);
if (rejected) return rejected;

const outcome = await Sentry.withMonitor(
"reap-retention",
() =>
reapCloudWorkspaceRetention({
production: process.env.VERCEL_ENV === "production",
}),
{ schedule: { type: "crontab", value: "0 */6 * * *" } },
);
// Turbopack builds do not wrap route handlers, so nothing else flushes the closing check-in.
await Sentry.flush(2_000).catch(() => false);
return Response.json({ ok: true, ...outcome });
}
33 changes: 33 additions & 0 deletions apps/api/src/app/api/cloud-workspaces/teardown/route.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
import { teardownSandbox } from "@superset/trpc/cloud-workspace-teardown";
import { z } from "zod";
import { verifyQstashRequest } from "@/lib/verifyQstash";

export const dynamic = "force-dynamic";
export const maxDuration = 60;

const payloadSchema = z
.object({
providerSandboxId: z.string().min(1),
driveName: z.null(),
attempt: z.number().int().min(0),
})
.strict();

/** Stops a box, then deletes it with its snapshots; requeues itself on failure. */
export async function POST(request: Request): Promise<Response> {
const body = await request.text();
const rejected = await verifyQstashRequest(
request,
body,
"/api/cloud-workspaces/teardown",
);
if (rejected) return rejected;

const parsed = payloadSchema.safeParse(JSON.parse(body));
if (!parsed.success) {
console.error("[cloud-workspaces/teardown] invalid payload", parsed.error);
return Response.json({ error: "Invalid payload" }, { status: 400 });
}
const outcome = await teardownSandbox(parsed.data);
return Response.json({ ok: true, outcome });
}
36 changes: 36 additions & 0 deletions docs/cloud-sandbox-mismatches.md
Original file line number Diff line number Diff line change
Expand Up @@ -783,6 +783,42 @@ sandbox is gone or unresumable is marked `failed` by the next `access`, so it
gets the failed screen and a Remove button rather than a sidebar entry that
never opens.

**Vercel's snapshot expiry was silent, so we own retention now.** The app
assumed a stopped box keeps its disk until someone deletes it. In fact Vercel
expired a workspace box's snapshot 30 days after its last use, and nothing told
us: the row learned only from the 410 its next wake got. New workspace boxes
are now created with `snapshotExpiration: 0`, and one cleanup job
(`router/cloud-workspace/retention.ts`, every 6 hours, production only) is the
only thing that deletes for age. It applies three rules from one table:

1. A workspace not in use is retired after its wait: asleep 30 days after last
use, archived 7 days, failed 30 days, stuck creating with no boot 1 day.
Retiring is one conditional write (still in that status, still past the
wait) that takes the box name off the row, then the retried teardown job
deletes the box. A wake that lands first wins. An unarchive after the 7 days
is refused even before the job runs.
2. An environment keeps its active golden and its newest others, 3 in all; an
archived environment keeps none. On main an environment records only its
active golden, so the job deletes archived environments' goldens.
3. A box, drive or golden no production row names goes 7 days after it was
made; an agents drive goes once it is not the current release's and no box,
stopped ones included, mounts it.

Everything else only sets a status: archive (its 60 s stop stays), a failed
create and a 410 on wake. Only production deletes: `sandboxDeletesAllowed`
(`lib/sandbox/deletes.ts`), checked by the teardown job, the drive delete and a
replaced golden's delete, logs "would delete" unless `VERCEL_ENV` is
`production`. `deleteSandbox` itself stays unchecked so a job still deletes the
temporary boxes it made (the release probe, real-sandbox). This is because
every deployment shares one Vercel project and a preview database copied from
production names production's boxes. Staging deploys as production on the
production database, so it counts. The job reads the whole box and drive
listing first and deletes nothing if it cannot. Boxes created before this
change keep a 30-day expiry until `bun run backfill-snapshot-expiration
--apply` (`packages/sandbox`) is run by hand. **Open:** no row names a
workspace drive on main; #9847's `drive_name` and `golden_drive_name` must
join rule 3's names when it lands.

**Any call on a `Sandbox` object resumes a stopped session.** The SDK wraps
`runCommand`, `writeFiles`, `extendTimeout` and the rest in `withResume`: on a
"stopped" answer it resumes and retries. A box read `running` that hits its
Expand Down
1 change: 1 addition & 0 deletions packages/sandbox/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
"pin": "bun run src/pin.ts",
"assets": "bun run src/assets/produce.ts",
"inventory": "bun run src/inventory.ts",
"backfill-snapshot-expiration": "bun run src/backfill-snapshot-expiration.ts",
"release": "bun run src/environments/release.ts",
"agent-tools": "bun run src/agent-tools.ts",
"record-bundle": "bun run src/environments/record-bundle.ts",
Expand Down
56 changes: 0 additions & 56 deletions packages/sandbox/src/agent-tools.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,18 +10,15 @@ import { tmpdir } from "node:os";
import { join } from "node:path";
import {
type AgentToolsManifest,
agentToolsDrivesToPrune,
agentToolsGaps,
agentToolsManifest,
type DriveHost,
describeAgentToolsPublish,
parseAgentToolsManifest,
parseBuilderResults,
pruneAgentToolsDrives,
publishAgentToolsDrive,
publishAgentToolsDrives,
type RegionPublish,
referencedAgentToolsDrives,
renderBuilderScript,
} from "./agent-tools";
import { type Assets, loadAssets, sha256 } from "./manifest";
Expand Down Expand Up @@ -387,59 +384,6 @@ describe("agentToolsGaps", () => {
});
});

test("referenced drives: every row's bundle in its region, and the recent bundles in every region", () => {
const older = "1".repeat(64);
expect(
[
...referencedAgentToolsDrives({
rows: [
{ region: "iad1", bundleSha: older },
{ region: "fra1", bundleSha: null },
],
recentBundles: [BUNDLE],
regions: ["sfo1", "iad1"],
}),
].sort(),
).toEqual([
"agent-tools-iad1-0123456789ab",
"agent-tools-iad1-111111111111",
"agent-tools-sfo1-0123456789ab",
]);
});

describe("pruning", () => {
test("deletes only agent-tools drives no environment references", () => {
expect(
agentToolsDrivesToPrune(
[
"agent-tools-sfo1-0123456789ab",
"agent-tools-sfo1-aaaaaaaaaaaa",
"agent-tools-iad1-0123456789ab",
"agent-tools-sfo1",
"agent-tools-builder-sfo1-xyz",
],
new Set(["agent-tools-sfo1-0123456789ab"]),
),
).toEqual([
"agent-tools-sfo1-aaaaaaaaaaaa",
"agent-tools-iad1-0123456789ab",
]);
});

test("keeps a drive a sandbox holds for the next release", async () => {
const { host } = fakeHost({
"agent-tools-sfo1-aaaaaaaaaaaa": {},
"agent-tools-sfo1-bbbbbbbbbbbb": { attachedTo: "box" },
});
expect(
await pruneAgentToolsDrives({ host, referenced: new Set() }),
).toEqual({
deleted: ["agent-tools-sfo1-aaaaaaaaaaaa"],
kept: ["agent-tools-sfo1-bbbbbbbbbbbb"],
});
});
});

async function runScript(script: string) {
const proc = Bun.spawn(["bash", "-c", script], {
stdout: "pipe",
Expand Down
52 changes: 0 additions & 52 deletions packages/sandbox/src/agent-tools.ts
Original file line number Diff line number Diff line change
Expand Up @@ -452,58 +452,6 @@ export function agentToolsGaps(
});
}

/**
* Every drive a box may still mount: each environment row's bundle in its
* region, and the newest released bundles in every region, so a rollback and
* an environment created mid-release still find theirs.
*/
export function referencedAgentToolsDrives(args: {
rows: readonly { region: string; bundleSha: string | null }[];
recentBundles: readonly string[];
regions: readonly string[];
}): Set<string> {
return new Set([
...args.rows.flatMap((row) =>
row.bundleSha ? [sandboxAgentToolsDrive(row.region, row.bundleSha)] : [],
),
...args.recentBundles.flatMap((sha) =>
args.regions.map((region) => sandboxAgentToolsDrive(region, sha)),
),
]);
}

/** Agent-tools drives no environment row's bundle and region name any more. */
export function agentToolsDrivesToPrune(
drives: readonly string[],
referenced: ReadonlySet<string>,
): string[] {
return drives.filter(
(name) =>
SANDBOX_AGENT_TOOLS_DRIVE_PATTERN.test(name) && !referenced.has(name),
);
}

/** Returns the drives it could not delete; a drive a sandbox holds is retried next release. */
export async function pruneAgentToolsDrives(args: {
host: DriveHost;
referenced: ReadonlySet<string>;
}): Promise<{ deleted: string[]; kept: string[] }> {
const deleted: string[] = [];
const kept: string[] = [];
for (const name of agentToolsDrivesToPrune(
(await args.host.list("agent-tools-")).map((drive) => drive.name),
args.referenced,
)) {
try {
await args.host.remove(name);
deleted.push(name);
} catch {
kept.push(name);
}
}
return { deleted, kept };
}

interface VercelCredentials {
token: string;
teamId: string;
Expand Down
39 changes: 39 additions & 0 deletions packages/sandbox/src/backfill-snapshot-expiration.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
import { expect, test } from "bun:test";
import { backfillSnapshotExpiration } from "./backfill-snapshot-expiration";

const boxes = [
{ name: "ws-default", snapshotExpiration: undefined },
{ name: "ws-30d", snapshotExpiration: 30 * 24 * 60 * 60 * 1000 },
{ name: "ws-done", snapshotExpiration: 0 },
];

test("changes only boxes that still expire, paced, and only with --apply", async () => {
const updated: string[] = [];
const slept: number[] = [];
const host = {
listWorkspaceBoxes: async () => boxes,
neverExpire: async (name: string) => {
updated.push(name);
},
sleep: async (ms: number) => {
slept.push(ms);
},
};
const dry = await backfillSnapshotExpiration({
host,
apply: false,
perSecond: 2,
log: () => {},
});
expect(dry).toMatchObject({ pending: 2, updated: 0 });
expect(updated).toEqual([]);

await backfillSnapshotExpiration({
host,
apply: true,
perSecond: 2,
log: () => {},
});
expect(updated).toEqual(["ws-default", "ws-30d"]);
expect(slept).toEqual([500]);
});
Loading
Loading