diff --git a/.env.example b/.env.example index b003095385..90157511eb 100644 --- a/.env.example +++ b/.env.example @@ -158,6 +158,7 @@ GITLAB_NAMESPACE= # --------------------------------------------------------------------------- SANDBOX_PROVIDER=modal +# Set modal-vm for deployment-wide Docker-capable VMs; see docs/MODAL_DOCKER.md. # Modal. The API secret is shared with the Modal deployment for HMAC-signed # endpoint calls; the workspace and environment build the endpoint URLs. diff --git a/.github/workflows/terraform.yml b/.github/workflows/terraform.yml index fe641767c8..a1aedc2d63 100644 --- a/.github/workflows/terraform.yml +++ b/.github/workflows/terraform.yml @@ -122,6 +122,13 @@ jobs: run: terraform test working-directory: ${{ env.TF_WORKING_DIR }} + - name: Modal module contract tests + id: modal_test + run: | + terraform init -backend=false + terraform test + working-directory: terraform/modules/modal-app + - name: Post Validation Results if: always() && github.event_name == 'pull_request' uses: actions/github-script@v8 @@ -140,6 +147,7 @@ jobs: | Init | ${{ steps.init.outcome == 'success' && '✅' || '❌' }} | | Validate | ${{ steps.validate.outcome == 'success' && '✅' || '❌' }} | | Tests | ${{ steps.test.outcome == 'success' && '✅' || '❌' }} | + | Modal module tests | ${{ steps.modal_test.outcome == 'success' && '✅' || '❌' }} | ${planNote} *Pushed by: @${{ github.actor }}, Action: \`${{ github.event_name }}\`*`; diff --git a/docs/GETTING_STARTED.md b/docs/GETTING_STARTED.md index 84542406b4..3606c5fe49 100644 --- a/docs/GETTING_STARTED.md +++ b/docs/GETTING_STARTED.md @@ -160,8 +160,9 @@ Create an R2 API Token: ### Modal -> Only required when `sandbox_provider = "modal"` (the default, used by the core path). To use -> Daytona, Vercel Sandboxes, OpenComputer, or E2B instead, skip this section and follow +> Only required when `sandbox_provider = "modal"` (the default, used by the core path) or +> `"modal-vm"`. Select `modal-vm` for Docker-capable VMs; see [Modal VM setup](MODAL_DOCKER.md). To +> use Daytona, Vercel Sandboxes, OpenComputer, or E2B instead, skip this section and follow > [Alternative Sandbox Providers](#alternative-sandbox-providers-optional). 1. Go to [Modal Settings](https://modal.com/settings) diff --git a/docs/MODAL_DOCKER.md b/docs/MODAL_DOCKER.md new file mode 100644 index 0000000000..c4cdbf8e60 --- /dev/null +++ b/docs/MODAL_DOCKER.md @@ -0,0 +1,76 @@ +# Modal VM backend + +Open Inspect offers two Modal compute backends: + +| `SANDBOX_PROVIDER` | Runtime | User Docker workloads | +| ------------------ | ----------------------- | --------------------- | +| `modal` (default) | Existing gVisor sandbox | Not enabled | +| `modal-vm` | Modal VM | Included | + +Selection is deployment-wide. There is no Docker checkbox or repository/session override. Mixed use +within one deployment is a follow-up. Both backends share a Modal app, account, credentials, +transport, and implementation; they have separate prepared-image pools. + +## Deployment + +For Terraform, set `sandbox_provider = "modal-vm"`. The existing Modal module builds and verifies +the Docker image before deploying the worker. No extra provisioning/admission flag is required. For +a standalone control plane, set `SANDBOX_PROVIDER=modal-vm` and provision the data plane first: + +```bash +cd packages/modal-infra +BUILD_MODAL_VM_IMAGE=true uv run python deploy.py --build-sandbox-image +BUILD_MODAL_VM_IMAGE=true uv run modal deploy deploy.py +``` + +These commands create billable resources. Use the intended Modal environment and credentials. Never +deploy `src/app.py` directly. VM launches reject old or incompatible API responses rather than +falling back to standard sandboxes. Existing standard clients can still use the standard endpoints. + +## Runtime and resources + +The harness, bridge, workspace, IDE, and desktop run directly on the VM host. Docker is for user +workloads such as PostgreSQL, Redis, and container builds. The runtime supervises the local daemon; +user environment variables cannot enable Docker or redirect its readiness probes. + +Generic `cpuCores` and `memoryMib` size the **outer VM**, not individual containers. Missing/null +values select the offering defaults (currently 2 cores and 4096 MiB). Positive explicit settings +override them. VM image builds use their scope's configured resources. Standard build sizing is +unchanged. These values are product defaults, not claimed Modal minimums. + +## Snapshots and recovery + +VM session snapshots are **destructive**: quiesce Docker, capture the filesystem, then confirm VM +termination. Later work restores into a new VM. Standard Modal snapshots remain non-destructive. A +failed or ambiguous capture/retirement must not be reported as a successful checkpoint. + +VMs therefore keep running between turns. They are saved and stopped on inactivity, lifetime expiry, +a lost heartbeat, a runtime failure or archive; a VM that stops heartbeating is captured without its +runtime. Cancelling a session stops its VM without saving. While a save has failed or its result is +unknown, new prompts are held and the session offers to retry the save, restore the last saved +state, or discard the VM and start fresh. + +Filesystem capture is not process/RAM continuity or an application-consistent database backup. +Containers must use appropriate persistence and restart policies. Live Docker pause/resume is not +provided. Raw daemon logs are truncated after clean preparation before reusable image capture. + +Retried VM launches adopt only an exactly owned allocation and recover its original interactive +credentials. A predecessor must be confirmed terminated before launching a replacement. Build +allocations have deterministic backend/build names, so a retried create adopts the allocation an +earlier lost response created. Returned build handles are persisted for cleanup before backend +validation; incompatible builds never start and cannot publish prepared images. + +## Switching backends + +Changing `SANDBOX_PROVIDER` is an operator cutover, not session migration. Existing sessions and +snapshots may become unusable. Drain/retire active allocations first when feasible, retain +credentials for pending cleanup, and rebuild images under the selected backend. Do not relabel old +artifacts. Rollback to `modal` does not transparently resume VM sessions or prove old VMs have +stopped. + +PR #2007's earlier per-session Docker/variant design was not deployed. Its schema additions and +settings are not part of this implementation, so no variant migration is required. + +Provider-backed canaries must validate Docker startup, build/restore, access after adoption, +snapshot/retirement, and cleanup in the target deployment before production rollout. Unit tests +alone do not prove those provider behaviors. diff --git a/packages/control-plane/src/image-builds/modal-adapter.test.ts b/packages/control-plane/src/image-builds/modal-adapter.test.ts index 866bb71b81..ddd0140040 100644 --- a/packages/control-plane/src/image-builds/modal-adapter.test.ts +++ b/packages/control-plane/src/image-builds/modal-adapter.test.ts @@ -41,6 +41,10 @@ function createPlan(): ImageBuildPlan { } describe("ModalImageBuildAdapter", () => { + it("does not advertise unbound source recovery", () => { + expect("recoverUnboundSource" in new ModalImageBuildAdapter(createProvider())).toBe(false); + }); + it("delegates build startup to the Modal provider", async () => { const provider = createProvider(); const adapter = new ModalImageBuildAdapter(provider); diff --git a/packages/control-plane/src/image-builds/modal-adapter.ts b/packages/control-plane/src/image-builds/modal-adapter.ts index ecc9050888..fabdf4b0b1 100644 --- a/packages/control-plane/src/image-builds/modal-adapter.ts +++ b/packages/control-plane/src/image-builds/modal-adapter.ts @@ -23,6 +23,7 @@ export class ModalImageBuildAdapter implements ImageBuildAdapter { await this.provider.triggerImageBuild({ scopeKind: plan.scope.kind, scopeId: plan.scope.id, + resources: plan.resources, buildId: plan.buildId, repositories: plan.repositories, cloneToken: plan.cloneAuth.type === "credential_helper" ? plan.cloneAuth.token : undefined, diff --git a/packages/control-plane/src/image-builds/model.ts b/packages/control-plane/src/image-builds/model.ts index 81667bfbc2..246cae2482 100644 --- a/packages/control-plane/src/image-builds/model.ts +++ b/packages/control-plane/src/image-builds/model.ts @@ -13,9 +13,10 @@ import { formatRepositoryFullName, parseRepositoryFullName, } from "@open-inspect/shared/types/repositories"; -import type { - ImageBuildScopeKind, - ImageBuildStatus, +import { + IMAGE_BUILD_PROVIDER_IDS, + type ImageBuildScopeKind, + type ImageBuildStatus, } from "@open-inspect/shared/types/image-builds"; import type { HarnessId } from "@open-inspect/shared/harnesses"; import { z } from "zod"; @@ -24,21 +25,7 @@ import { MIN_COMPATIBLE_RUNTIME_GENERATION, } from "../sandbox/runtime-manifest"; -/** - * Providers with image-build support: Modal images, Vercel snapshots, - * OpenComputer checkpoints, E2B snapshots, Daytona snapshots. - * - * Support is the provider's ability to build and boot an artifact. Whether a - * deployment may start new Daytona builds is a separate, operator-owned - * question — see `resolveImageBuildAdmission` in provider-policy.ts. - */ -export const IMAGE_BUILD_PROVIDER_IDS = [ - "modal", - "vercel", - "opencomputer", - "e2b", - "daytona", -] as const; +export { IMAGE_BUILD_PROVIDER_IDS } from "@open-inspect/shared/types/image-builds"; export const imageBuildProviderSchema = z.enum(IMAGE_BUILD_PROVIDER_IDS); diff --git a/packages/control-plane/src/image-builds/planner.ts b/packages/control-plane/src/image-builds/planner.ts index 7fab22e9fc..9a61f7fa8d 100644 --- a/packages/control-plane/src/image-builds/planner.ts +++ b/packages/control-plane/src/image-builds/planner.ts @@ -99,6 +99,7 @@ export class ImageBuildPlanner implements ImageBuildPlannerPort { callbackUrl: params.callbackUrl, failureCallbackUrl: params.failureCallbackUrl, buildTimeoutMs: resolveBuildTimeoutSeconds(sandboxSettings) * MS_PER_SECOND, + resources: { cpuCores: sandboxSettings.cpuCores, memoryMib: sandboxSettings.memoryMib }, userEnvVars: userEnvVars ? prepareLegacyManagedProviderEnv({ exposedSecrets: userEnvVars, diff --git a/packages/control-plane/src/image-builds/provider-factory.ts b/packages/control-plane/src/image-builds/provider-factory.ts index e91efa682d..0c9535e110 100644 --- a/packages/control-plane/src/image-builds/provider-factory.ts +++ b/packages/control-plane/src/image-builds/provider-factory.ts @@ -37,7 +37,8 @@ class EnvImageBuildAdapterFactory implements ImageBuildAdapterFactory { create(provider: ImageBuildProvider, operation: "start" | "existing_session"): ImageBuildAdapter { switch (provider) { case "modal": - return new ModalImageBuildAdapter(createSandboxProviderFromEnv(this.env, "modal")); + case "modal-vm": + return new ModalImageBuildAdapter(createSandboxProviderFromEnv(this.env, provider)); case "vercel": return new VercelImageBuildAdapter(createSandboxProviderFromEnv(this.env, "vercel")); case "opencomputer": diff --git a/packages/control-plane/src/image-builds/types.ts b/packages/control-plane/src/image-builds/types.ts index 9ba09a11ce..cc06769080 100644 --- a/packages/control-plane/src/image-builds/types.ts +++ b/packages/control-plane/src/image-builds/types.ts @@ -1,4 +1,5 @@ import type { RepositoryShaEntry } from "@open-inspect/shared/types/image-builds"; +import type { SandboxSettings } from "@open-inspect/shared/types/integrations"; import type { CorrelationContext } from "../logger"; import type { ImageBuildProviderImageRef, ImageBuildScope } from "./model"; @@ -32,6 +33,7 @@ export type ImageBuildCloneAuth = * Every supported provider uses the same create-bind-launch session contract. */ export interface ImageBuildPlan { + resources?: Pick; buildId: string; scope: ImageBuildScope; repositories: ImageBuildRepository[]; diff --git a/packages/control-plane/src/routes/image-builds.trigger.test.ts b/packages/control-plane/src/routes/image-builds.trigger.test.ts index 0d8f68bac1..878bb68fc2 100644 --- a/packages/control-plane/src/routes/image-builds.trigger.test.ts +++ b/packages/control-plane/src/routes/image-builds.trigger.test.ts @@ -205,6 +205,7 @@ const registerBuildSpy = vi.spyOn(ImageBuildStore.prototype, "registerBuild"); const getActiveBuildSpy = vi.spyOn(ImageBuildStore.prototype, "getActiveBuild"); const hasReadyImageSpy = vi.spyOn(ImageBuildStore.prototype, "hasReadyImageForFingerprint"); const markBuildFailedSpy = vi.spyOn(ImageBuildStore.prototype, "markBuildFailed"); +const markSourceCreateIntentSpy = vi.spyOn(ImageBuildStore.prototype, "markSourceCreateIntent"); const bindProviderSessionSpy = vi.spyOn(ImageBuildStore.prototype, "bindProviderSession"); const setImageBuildEnabledSpy = vi.spyOn(RepoMetadataStore.prototype, "setImageBuildEnabled"); @@ -220,6 +221,7 @@ beforeEach(() => { markBuildFailedSpy.mockResolvedValue(true); setImageBuildEnabledSpy.mockResolvedValue(undefined); bindProviderSessionSpy.mockResolvedValue(true); + markSourceCreateIntentSpy.mockResolvedValue(true); modalClient.createImageBuildSandbox.mockResolvedValue({ providerSessionId: "modal-session-1", }); diff --git a/packages/control-plane/src/sandbox/client.ts b/packages/control-plane/src/sandbox/client.ts index 70c4cd54e0..1d4d9240f7 100644 --- a/packages/control-plane/src/sandbox/client.ts +++ b/packages/control-plane/src/sandbox/client.ts @@ -16,6 +16,8 @@ import { buildSessionConfig, toRepositoryConfigPayload } from "./sandbox-env"; import type { SessionRepositoryInfo } from "./provider"; import { withRequestDeadline } from "./request-deadline"; +export type ModalBackend = "modal" | "modal-vm"; + const log = createLogger("modal-client"); // Modal app name @@ -36,6 +38,7 @@ const createSandboxModalResponseSchema = z.object({ data: z.object({ sandbox_id: z.string(), modal_object_id: z.string().nullable().optional(), + sandbox_backend: z.unknown().optional(), created_at: z.number(), code_server_url: z.string().nullable().optional(), code_server_password: z.string().nullable().optional(), @@ -51,6 +54,7 @@ const restoreSandboxModalResponseSchema = z.object({ data: z.object({ sandbox_id: z.string().min(1), modal_object_id: z.string().nullable().optional(), + sandbox_backend: z.unknown().optional(), code_server_url: z.string().nullable().optional(), code_server_password: z.string().nullable().optional(), vnc_url: z.string().nullable().optional(), @@ -64,6 +68,8 @@ const snapshotSandboxModalResponseSchema = z.object({ success: z.literal(true), data: z.object({ image_id: z.string().min(1), + source_stopped: z.boolean().optional(), + source_id: z.string().min(1).optional(), }), }); @@ -72,6 +78,7 @@ const createImageBuildSandboxModalResponseSchema = z.object({ data: z.object({ // Non-empty: the previous hand-rolled check rejected a blank id. provider_session_id: z.string().min(1), + sandbox_backend: z.unknown().optional(), }), }); @@ -141,6 +148,8 @@ export function buildModalSandboxDashboardUrl(params: { } export interface CreateSandboxRequest { + sandboxBackend?: ModalBackend; + retireSandboxId?: string | null; sessionId: string; sandboxId?: string; // Expected sandbox ID (generated by control plane) repoOwner: string | null; @@ -166,6 +175,8 @@ export interface CreateSandboxRequest { } export interface CreateSandboxResponse { + /** Validated by the provider after retaining the allocation handle. */ + sandboxBackend?: unknown; sandboxId: string; modalObjectId?: string; // Modal's internal object ID for snapshot API createdAt: number; @@ -178,6 +189,8 @@ export interface CreateSandboxResponse { } export interface RestoreSandboxRequest { + sandboxBackend?: ModalBackend; + retireSandboxId?: string | null; snapshotImageId: string; sessionId: string; sandboxId: string; @@ -201,6 +214,8 @@ export interface RestoreSandboxRequest { } export interface RestoreSandboxResponse { + /** Validated by the provider after retaining the allocation handle. */ + sandboxBackend?: unknown; sandboxId: string; modalObjectId?: string; codeServerUrl?: string; @@ -214,6 +229,7 @@ export interface RestoreSandboxResponse { export interface SnapshotSandboxRequest { providerObjectId: string; sessionId: string; + sandboxBackend?: ModalBackend; signal?: AbortSignal; deadlineAtMs?: number; } @@ -225,6 +241,8 @@ export interface StopSandboxRequest { } export interface SnapshotSandboxResponse { + sourceStopped?: boolean; + sourceObjectId?: string; imageId: string; } @@ -235,6 +253,8 @@ export interface SnapshotBuildSandboxRequest { } export interface CreateImageBuildSandboxRequest { + resources?: Pick; + sandboxBackend?: ModalBackend; /** Scope kind ("repo" | "environment") — accepted by Modal for logging only. */ scopeKind: ImageBuildScopeKind; /** Scope id (lowercase owner/name or environment id) — logging only. */ @@ -255,6 +275,8 @@ export interface CreateImageBuildSandboxRequest { } export interface CreateImageBuildSandboxResponse { + /** Validated by the provider after retaining the allocation handle. */ + sandboxBackend?: unknown; providerSessionId: string; } @@ -294,6 +316,7 @@ export class ModalApiError extends Error { export class ModalClient { private createSandboxUrl: string; private snapshotSandboxUrl: string; + private snapshotVmSandboxUrl: string; private snapshotBuildSandboxUrl: string; private restoreSandboxUrl: string; private stopSandboxUrl: string; @@ -341,6 +364,7 @@ export class ModalClient { modalEndpointUrl(functionName, workspace, environmentWebSuffix, apiUrl); this.createSandboxUrl = url("api-create-sandbox"); this.snapshotSandboxUrl = url("api-snapshot-sandbox"); + this.snapshotVmSandboxUrl = url("api-snapshot-vm-sandbox"); this.snapshotBuildSandboxUrl = url("api-snapshot-build-sandbox"); this.restoreSandboxUrl = url("api-restore-sandbox"); this.stopSandboxUrl = url("api-stop-sandbox"); @@ -403,6 +427,8 @@ export class ModalClient { agent_slack_notify_enabled: request.agentSlackNotifyEnabled ?? false, mcp_servers: request.mcpServers || null, sandbox_settings: request.sandboxSettings ?? null, + sandbox_backend: request.sandboxBackend, + retire_sandbox_id: request.retireSandboxId, // Flat keys matching SessionConfig field names — Modal's create // handler builds its SessionConfig from the request by field name // (unlike restore, which carries a nested session_config). @@ -421,6 +447,7 @@ export class ModalClient { return { sandboxId: result.data.sandbox_id, modalObjectId: result.data.modal_object_id ?? undefined, + sandboxBackend: result.data.sandbox_backend, createdAt: result.data.created_at, codeServerUrl: result.data.code_server_url ?? undefined, codeServerPassword: result.data.code_server_password ?? undefined, @@ -473,6 +500,8 @@ export class ModalClient { vnc_enabled: request.vncEnabled ?? false, agent_slack_notify_enabled: request.agentSlackNotifyEnabled ?? false, sandbox_settings: request.sandboxSettings ?? null, + sandbox_backend: request.sandboxBackend, + retire_sandbox_id: request.retireSandboxId, }, restoreSandboxModalResponseSchema, correlation, @@ -484,6 +513,7 @@ export class ModalClient { return { sandboxId: result.data.sandbox_id, modalObjectId: result.data.modal_object_id ?? undefined, + sandboxBackend: result.data.sandbox_backend, codeServerUrl: result.data.code_server_url ?? undefined, codeServerPassword: result.data.code_server_password ?? undefined, vncUrl: result.data.vnc_url ?? undefined, @@ -506,9 +536,7 @@ export class ModalClient { } } - /** - * Trigger a filesystem snapshot for a sandbox object. - */ + /** Trigger a filesystem snapshot for a sandbox object. */ async snapshotSandbox( request: SnapshotSandboxRequest, correlation?: CorrelationContext @@ -520,7 +548,7 @@ export class ModalClient { try { const result = await this.postJson( - this.snapshotSandboxUrl, + request.sandboxBackend === "modal-vm" ? this.snapshotVmSandboxUrl : this.snapshotSandboxUrl, endpoint, request.deadlineAtMs === undefined ? MODAL_SNAPSHOT_REQUEST_DEADLINE_MS @@ -531,6 +559,7 @@ export class ModalClient { { sandbox_id: request.providerObjectId, deadline_at_ms: request.deadlineAtMs ?? null, + ...(request.sandboxBackend ? { sandbox_backend: request.sandboxBackend } : {}), }, snapshotSandboxModalResponseSchema, correlation, @@ -538,7 +567,11 @@ export class ModalClient { (status) => (httpStatus = status) ); outcome = "success"; - return { imageId: result.data.image_id }; + return { + imageId: result.data.image_id, + sourceStopped: result.data.source_stopped, + sourceObjectId: result.data.source_id, + }; } finally { log.info("modal.request", { event: "modal.request", @@ -594,7 +627,7 @@ export class ModalClient { (status) => (httpStatus = status) ); outcome = "success"; - return { imageId: result.data.image_id }; + return { imageId: result.data.image_id, sourceStopped: result.data.source_stopped }; } finally { log.info("modal.request", { event: "modal.request", @@ -625,6 +658,8 @@ export class ModalClient { endpoint, MODAL_SANDBOX_START_REQUEST_DEADLINE_MS, { + sandbox_backend: request.sandboxBackend, + sandbox_settings: request.resources, scope_kind: request.scopeKind, scope_id: request.scopeId, build_id: request.buildId, @@ -647,6 +682,7 @@ export class ModalClient { outcome = "success"; return { providerSessionId: result.data.provider_session_id, + sandboxBackend: result.data.sandbox_backend, }; } finally { log.info("modal.request", { diff --git a/packages/control-plane/src/sandbox/lifecycle/alarm-heartbeat-effects.test.ts b/packages/control-plane/src/sandbox/lifecycle/alarm-heartbeat-effects.test.ts index e76fd77912..9d4136a29e 100644 --- a/packages/control-plane/src/sandbox/lifecycle/alarm-heartbeat-effects.test.ts +++ b/packages/control-plane/src/sandbox/lifecycle/alarm-heartbeat-effects.test.ts @@ -64,6 +64,78 @@ describe("heartbeat alarm effects", () => { } ); + it("saves a stale sandbox whose snapshots stop it without its runtime, then stops it", async () => { + const sandbox = createMockSandbox({ status: "ready", last_heartbeat: Date.now() - 100_000 }); + const takeSnapshot = vi.fn(async () => ({ + success: true, + imageId: "heartbeat-image", + sourceStopped: false, + })); + const stopSandbox = vi.fn(async () => ({ success: true })); + const h = createAlarmFixture( + sandbox, + createMockProvider({ + capabilities: { supportsExplicitStop: true, snapshotRequiresShutdown: true }, + takeSnapshot, + stopSandbox, + }) + ); + + await expect(h.manager.handleAlarm()).resolves.toBe("no_action"); + + expect(takeSnapshot).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining({ reason: "heartbeat_timeout" }) + ); + expect(stopSandbox).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining({ reason: "heartbeat_timeout", intent: "destroy" }) + ); + expect(takeSnapshot.mock.invocationCallOrder[0]).toBeLessThan( + stopSandbox.mock.invocationCallOrder[0] + ); + // No graceful drain: the runtime is never asked to prepare or shut down. + expect(h.wsManager.sendToSandbox).not.toHaveBeenCalled(); + expect(sandbox.snapshot_image_id).toBe("heartbeat-image"); + expect(sandbox.status).toBe("stopped"); + expect(h.broadcaster.messages).toContainEqual({ type: "sandbox_status", status: "stale" }); + expect(h.broadcaster.messages).toContainEqual( + expect.objectContaining({ + type: "sandbox_preservation", + preservation: expect.objectContaining({ phase: "saved", continuationPaused: true }), + }) + ); + }); + + it("keeps a stale sandbox for another attempt when saving it without its runtime fails", async () => { + const sandbox = createMockSandbox({ status: "ready", last_heartbeat: Date.now() - 100_000 }); + const stopSandbox = vi.fn(async () => ({ success: true })); + const h = createAlarmFixture( + sandbox, + createMockProvider({ + capabilities: { supportsExplicitStop: true, snapshotRequiresShutdown: true }, + takeSnapshot: vi.fn(async () => { + throw new Error("guest unresponsive"); + }), + stopSandbox, + }) + ); + + await expect(h.manager.handleAlarm()).resolves.toBe("no_action"); + + expect(stopSandbox).not.toHaveBeenCalled(); + expect(sandbox.status).toBe("stale"); + expect(h.broadcaster.messages).toContainEqual( + expect.objectContaining({ + type: "sandbox_preservation", + preservation: expect.objectContaining({ + phase: "unknown", + availableRecoveryActions: ["retry"], + discardAvailable: true, + }), + }) + ); + expect(h.manager.onRefusedReconnect()).toBe("retry"); + }); + it("does not await a heartbeat snapshot when the provider cannot explicitly stop", async () => { const sandbox = createMockSandbox({ last_heartbeat: Date.now() - 100_000 }); let releaseSnapshot!: (result: SnapshotResult) => void; diff --git a/packages/control-plane/src/sandbox/lifecycle/alarm-inactivity-effects.test.ts b/packages/control-plane/src/sandbox/lifecycle/alarm-inactivity-effects.test.ts index 7663e00e92..02a74e5c25 100644 --- a/packages/control-plane/src/sandbox/lifecycle/alarm-inactivity-effects.test.ts +++ b/packages/control-plane/src/sandbox/lifecycle/alarm-inactivity-effects.test.ts @@ -10,10 +10,10 @@ describe("inactivity alarm effects", () => { it.each([ { - name: "remaining inactivity", + name: "heartbeat before remaining inactivity", ageMs: 120_000, clients: 0, - delayMs: DEFAULT_LIFECYCLE_CONFIG.inactivity.timeoutMs - 120_000, + delayMs: DEFAULT_LIFECYCLE_CONFIG.heartbeat.timeoutMs + 1, }, { name: "minimum interval", @@ -140,7 +140,7 @@ describe("inactivity alarm effects", () => { sandbox, createMockProvider({ capabilities: { - snapshotStopsSandbox: true, + snapshotRequiresShutdown: true, supportsExplicitStop: true, supportsPersistentResume: false, }, diff --git a/packages/control-plane/src/sandbox/lifecycle/alarm-policy.test.ts b/packages/control-plane/src/sandbox/lifecycle/alarm-policy.test.ts index c7a3857cf5..eade4a4cb3 100644 --- a/packages/control-plane/src/sandbox/lifecycle/alarm-policy.test.ts +++ b/packages/control-plane/src/sandbox/lifecycle/alarm-policy.test.ts @@ -143,7 +143,7 @@ describe("evaluateAlarmPolicy", () => { evaluateAlarmPolicy(row({ status: "ready", last_heartbeat: now - ageMs }), config, now, 0) ).toEqual( offsetMs <= 0 - ? healthy + ? { outcome: "healthy", nextCheckMs: Math.max(1, -offsetMs + 1) } : { outcome: "heartbeat_stale", ageMs, @@ -161,7 +161,7 @@ describe("evaluateAlarmPolicy", () => { it.each<[number | null, number, AlarmFinding]>([ [null, 0, healthy], - [now - 300_000, 0, { outcome: "healthy", nextCheckMs: 300_000 }], + [now - 300_000, 0, { outcome: "healthy", nextCheckMs: 90_001 }], [now - 599_999, 0, healthy], [now - 600_000, 0, { outcome: "inactivity_timeout" }], [now - 600_001, 0, { outcome: "inactivity_timeout" }], @@ -184,6 +184,17 @@ describe("evaluateAlarmPolicy", () => { ).toEqual(expected); }); + it("checks heartbeat before the ten-minute inactivity deadline", () => { + expect( + evaluateAlarmPolicy( + row({ status: "ready", last_heartbeat: now - 30_000, last_activity: now }), + config, + now, + 0 + ) + ).toEqual({ outcome: "healthy", nextCheckMs: 60_001 }); + }); + it.each(["pending", "snapshotting"] as const)("does not stop an idle %s row", (status) => { expect( evaluateAlarmPolicy(row({ status, last_heartbeat: now, last_activity: 0 }), config, now, 0) diff --git a/packages/control-plane/src/sandbox/lifecycle/alarm-policy.ts b/packages/control-plane/src/sandbox/lifecycle/alarm-policy.ts index 05faed8b01..f232d314fb 100644 --- a/packages/control-plane/src/sandbox/lifecycle/alarm-policy.ts +++ b/packages/control-plane/src/sandbox/lifecycle/alarm-policy.ts @@ -82,6 +82,17 @@ export function evaluateAlarmPolicy( case "extend": return { outcome: "inactivity_warning", extensionMs: inactivity.extensionMs }; case "schedule": - return { outcome: "healthy", nextCheckMs: inactivity.nextCheckMs }; + // The inactivity deadline can be ten minutes away. A bridge that stops + // heartbeating just after this alarm must still be noticed at 90s. + return { + outcome: "healthy", + nextCheckMs: + sandbox.last_heartbeat === null + ? inactivity.nextCheckMs + : Math.min( + inactivity.nextCheckMs, + Math.max(1, sandbox.last_heartbeat + config.heartbeat.timeoutMs + 1 - now) + ), + }; } } diff --git a/packages/control-plane/src/sandbox/lifecycle/manager.test.ts b/packages/control-plane/src/sandbox/lifecycle/manager.test.ts index 9f6c5ca005..e6e694f890 100644 --- a/packages/control-plane/src/sandbox/lifecycle/manager.test.ts +++ b/packages/control-plane/src/sandbox/lifecycle/manager.test.ts @@ -22,6 +22,7 @@ import { PrebuiltImageActivationPendingError, PrebuiltImageUnavailableError, SandboxProviderError, + SandboxLaunchRejectedError, type SandboxProvider, type CreateSandboxConfig, type CreateSandboxResult, @@ -356,7 +357,10 @@ describe("final graceful shutdown lifecycle integration", () => { receipt: { artifactId: "saved-image" }, }); expect(restoreFromSnapshot).toHaveBeenCalledOnce(); - expect(saved.shutdown.snapshot()?.availableRecoveryActions).toEqual(["restore_saved"]); + expect(saved.shutdown.snapshot()).toMatchObject({ + availableRecoveryActions: ["restore_saved"], + discardAvailable: true, + }); await saved.shutdown.recover("restore_saved"); await f.manager.spawnSandbox(); expect(restoreFromSnapshot).toHaveBeenCalledTimes(2); @@ -385,24 +389,46 @@ describe("final graceful shutdown lifecycle integration", () => { expect(f.provider.createSandbox).not.toHaveBeenCalled(); }); - it("routes destructive ordinary snapshots through confirmed graceful shutdown", async () => { - const f = fixture(createMockProvider({ capabilities: { snapshotStopsSandbox: true } })); + it("saves an archived session's sandbox through graceful shutdown", async () => { + const f = fixture( + createMockProvider({ capabilities: { snapshotRequiresShutdown: true } }), + createMockSandbox({ status: "ready" }) + ); + await f.manager.preserveForArchive(); + expect(f.shutdown.requestShutdown).toHaveBeenCalledExactlyOnceWith("session_archived"); + }); + + it("keeps a sandbox whose snapshots stop it running after a finished turn", async () => { + const sandbox = createMockSandbox({ status: "ready" }); + const f = fixture( + createMockProvider({ capabilities: { snapshotRequiresShutdown: true } }), + sandbox + ); await f.manager.triggerSnapshot("execution_complete"); - expect(f.shutdown.requestShutdown).toHaveBeenCalledWith("execution_complete"); + expect(f.shutdown.requestShutdown).not.toHaveBeenCalled(); + expect(f.shutdown.captureCheckpoint).not.toHaveBeenCalled(); + expect(f.provider.takeSnapshot).not.toHaveBeenCalled(); + expect(sandbox.status).toBe("ready"); + }); + + it("routes other destructive snapshots through confirmed graceful shutdown", async () => { + const f = fixture(createMockProvider({ capabilities: { snapshotRequiresShutdown: true } })); + await f.manager.triggerSnapshot("inactivity_timeout"); + expect(f.shutdown.requestShutdown).toHaveBeenCalledWith("inactivity_timeout"); expect(f.provider.takeSnapshot).not.toHaveBeenCalled(); }); it("does not fall through to a destructive checkpoint when shutdown declines it", async () => { const sandbox = createMockSandbox({ status: "ready" }); const f = fixture( - createMockProvider({ capabilities: { snapshotStopsSandbox: true } }), + createMockProvider({ capabilities: { snapshotRequiresShutdown: true } }), sandbox ); f.shutdown.requestShutdown.mockResolvedValue("held"); - await f.manager.triggerSnapshot("execution_complete"); + await f.manager.triggerSnapshot("inactivity_timeout"); - expect(f.shutdown.requestShutdown).toHaveBeenCalledWith("execution_complete"); + expect(f.shutdown.requestShutdown).toHaveBeenCalledWith("inactivity_timeout"); expect(f.shutdown.captureCheckpoint).not.toHaveBeenCalled(); expect(f.provider.takeSnapshot).not.toHaveBeenCalled(); expect(sandbox.status).toBe("ready"); @@ -661,12 +687,23 @@ describe("final graceful shutdown lifecycle integration", () => { }); describe("lifecycle-owned runtime readiness and cancellation", () => { - function harness(status: SandboxStatus | null, attached = true) { + function harness( + status: SandboxStatus | null, + attached = true, + explicitStop = false, + persistentResume = false + ) { const row = status === null ? null : createMockSandbox({ status }); const storage = createMockStorage(createMockSession(), row); const broadcaster = createMockBroadcaster(); const ws = createMockWebSocketManager(attached); - const provider = createMockProvider({ stopSandbox: vi.fn(async () => ({ success: true })) }); + const provider = createMockProvider({ + stopSandbox: vi.fn(async () => ({ success: true })), + capabilities: { + supportsExplicitStop: explicitStop, + supportsPersistentResume: persistentResume, + }, + }); const alarms = createMockAlarmScheduler(); const manager = new SandboxLifecycleManager( provider, @@ -691,43 +728,77 @@ describe("lifecycle-owned runtime readiness and cancellation", () => { "snapshotting", "stale", ] as const)( - "preserves cancellation of %s without introducing provider retirement or fencing", - (status) => { - const h = harness(status); - h.manager.cancelSandbox(); + "cancels %s by asking the runtime to exit, then stopping it at the provider", + async (status) => { + const h = harness(status, true, true); + await h.manager.cancelSandbox(); expect(h.ws.sendToSandbox).toHaveBeenCalledWith({ type: "shutdown" }); expect(h.storage.updateSandboxStatus).toHaveBeenCalledWith("stopped"); expect(vi.mocked(h.ws.sendToSandbox).mock.invocationCallOrder[0]).toBeLessThan( vi.mocked(h.storage.updateSandboxStatus).mock.invocationCallOrder[0] ); - expect(h.provider.stopSandbox).not.toHaveBeenCalled(); + expect(h.provider.stopSandbox).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining({ + providerObjectId: "modal-obj-123", + reason: "session_cancelled", + intent: "destroy", + }) + ); + expect(vi.mocked(h.storage.updateSandboxStatus).mock.invocationCallOrder[0]).toBeLessThan( + vi.mocked(h.provider.stopSandbox!).mock.invocationCallOrder[0] + ); expect(h.storage.fenceSandboxGeneration).not.toHaveBeenCalled(); expect(h.ws.detachSandboxWebSocket).not.toHaveBeenCalled(); expect(h.broadcaster.messages).toEqual([]); } ); - it.each(["stopped", "failed", null] as const)("leaves %s unchanged on cancel", (status) => { - const h = harness(status); - h.manager.cancelSandbox(); + it("destroys a resumable provider's sandbox on cancel, since the session never resumes", async () => { + const h = harness("ready", true, true, true); + await h.manager.cancelSandbox(); + expect(h.provider.stopSandbox).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining({ reason: "session_cancelled", intent: "destroy" }) + ); + }); + + it("cancels without a provider stop when the provider cannot stop explicitly", async () => { + const h = harness("ready"); + await h.manager.cancelSandbox(); + expect(h.storage.updateSandboxStatus).toHaveBeenCalledWith("stopped"); + expect(h.provider.stopSandbox).not.toHaveBeenCalled(); + }); + + it("still records a cancellation when the provider stop fails", async () => { + const h = harness("ready", true, true); + vi.mocked(h.provider.stopSandbox!).mockRejectedValue(new Error("provider unavailable")); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + await h.manager.cancelSandbox(); + expect(h.row?.status).toBe("stopped"); + expect(warn).toHaveBeenCalledWith(expect.stringContaining("provider unavailable")); + }); + + it.each(["stopped", "failed", null] as const)("leaves %s unchanged on cancel", async (status) => { + const h = harness(status, true, true); + await h.manager.cancelSandbox(); expect(h.ws.getSandboxWebSocket).not.toHaveBeenCalled(); expect(h.storage.updateSandboxStatus).not.toHaveBeenCalled(); + expect(h.provider.stopSandbox).not.toHaveBeenCalled(); }); it.each(["ready", "stale"] as const)( "cancels a %s row without an attached dispatch socket", - (status) => { + async (status) => { const h = harness(status, false); - h.manager.cancelSandbox(); + await h.manager.cancelSandbox(); expect(h.ws.sendToSandbox).not.toHaveBeenCalled(); expect(h.storage.updateSandboxStatus).toHaveBeenCalledWith("stopped"); } ); - it("still records stopped when the existing local shutdown send fails", () => { + it("still records stopped when the existing local shutdown send fails", async () => { const h = harness("ready"); vi.mocked(h.ws.sendToSandbox).mockReturnValue(false); - h.manager.cancelSandbox(); + await h.manager.cancelSandbox(); expect(h.row?.status).toBe("stopped"); }); @@ -2879,7 +2950,7 @@ describe("SandboxLifecycleManager", () => { const storage = createMockStorage(createMockSession(), sandbox); const broadcaster = createMockBroadcaster(); const provider = createMockProvider({ - capabilities: { snapshotStopsSandbox: true }, + capabilities: { snapshotRequiresShutdown: true }, takeSnapshot: vi.fn(async () => ({ success: false, error: "capture failed" })), }); const shutdown = createCheckpointShutdown(provider, storage, broadcaster); @@ -4998,27 +5069,33 @@ describe("status writes after a provider await (COL-99)", () => { } }); - it("counts an attempt once when the watchdog fails it before the provider rejects it", async () => { - // The connect alarm is armed at reservation, before the provider call, - // so it can fail the attempt while createSandbox() is still pending. The - // provider's later rejection is the same attempt, not a second failure. - vi.useFakeTimers(); - try { - const sandbox = createMockSandbox({ status: "failed" }); - const h = harness(sandbox, async () => { - vi.advanceTimersByTime(DEFAULT_LIFECYCLE_CONFIG.connectingTimeout.timeoutMs + 1000); - await expect(h.manager.handleAlarm()).resolves.toBe("sandbox_failed"); - throw new SandboxProviderError("quota exceeded", "permanent"); - }); + it.each([ + new SandboxProviderError("quota exceeded", "permanent"), + new SandboxLaunchRejectedError("incompatible", "sb-rejected"), + ])( + "counts an attempt once when the watchdog fails it before the provider rejects it (%s)", + async (error) => { + // The connect alarm is armed at reservation, before the provider call, + // so it can fail the attempt while createSandbox() is still pending. The + // provider's later rejection is the same attempt, not a second failure. + vi.useFakeTimers(); + try { + const sandbox = createMockSandbox({ status: "failed" }); + const h = harness(sandbox, async () => { + vi.advanceTimersByTime(DEFAULT_LIFECYCLE_CONFIG.connectingTimeout.timeoutMs + 1000); + await expect(h.manager.handleAlarm()).resolves.toBe("sandbox_failed"); + throw error; + }); - await h.manager.spawnSandbox(); + await h.manager.spawnSandbox(); - expect(sandbox.status).toBe("failed"); - expect(sandbox.spawn_failure_count).toBe(1); - } finally { - vi.useRealTimers(); + expect(sandbox.status).toBe("failed"); + expect(sandbox.spawn_failure_count).toBe(1); + } finally { + vi.useRealTimers(); + } } - }); + ); it("leaves a sandbox whose bridge attached during the provider call booting when the call then fails", async () => { // The bridge now connects ahead of its boot, so a provider error that diff --git a/packages/control-plane/src/sandbox/lifecycle/manager.ts b/packages/control-plane/src/sandbox/lifecycle/manager.ts index 70e3283709..df4f60c862 100644 --- a/packages/control-plane/src/sandbox/lifecycle/manager.ts +++ b/packages/control-plane/src/sandbox/lifecycle/manager.ts @@ -34,6 +34,7 @@ import { import { PrebuiltImageUnavailableError, SandboxProviderError, + SandboxLaunchRejectedError, type SandboxProvider, type CreateSandboxConfig, type CreateSandboxResult, @@ -95,6 +96,7 @@ const log = createLogger("lifecycle-manager"); /** TTL for terminal auth JWTs (24 hours, matching typical sandbox lifetime). */ const TERMINAL_TOKEN_TTL_SECONDS = 86400; const PROVIDER_REPLACEMENT_STOP_TIMEOUT_MS = 10_000; +const REJECTED_ALLOCATION_CLEANUP_RETRY_MS = 30_000; // ==================== Dependency Interfaces ==================== @@ -112,6 +114,8 @@ export interface SandboxShutdownLifecycle { recordProviderStartup(generation: SandboxGeneration, lifetime: SandboxLifetime): Promise; /** Blocks generic destructive lifecycle work while shutdown or capture ownership is unresolved. */ isHolding(): boolean; + /** Tells a runtime refused at reconnect to retry while a capture needs its sandbox. */ + onRefusedReconnect(): "retry" | "exit"; /** Owns termination; only unmanaged permits the legacy lifecycle fallback. */ requestShutdown( reason: string, @@ -209,6 +213,14 @@ export interface SandboxStorage { from: SandboxStatus, to: SandboxStatus ): boolean; + /** + * Fence a rejected generation and retain its cleanup handle without replacing + * a terminal status. Superseded generations cannot modify the current row. + */ + rejectProviderStartup( + generation: SandboxGeneration, + providerObjectId: string | null + ): "failed" | "retained" | "superseded"; /** * Atomically accept a provider startup result for the named generation, * store its handle, and advance a fresh spawn to connecting. Returns the @@ -747,6 +759,10 @@ export class SandboxLifecycleManager } const hasRepository = sessionHasRepository(session); + const priorSandbox = this.storage.getSandbox(); + const priorSandboxId = priorSandbox?.modal_sandbox_id ?? null; + // A fenced allocation must be retired before its durable identity is replaced. + if (priorSandbox?.fenced) await this.stopPriorProviderSandbox(true); const reserved = this.spawnGeneration(session, now); generation = reserved; let { sandboxAuthToken, expectedSandboxId } = await this.reserveSpawnIdentity(reserved, { @@ -796,6 +812,7 @@ export class SandboxLifecycleManager const timeoutSeconds = this.resolveSandboxTimeoutSeconds(sandboxSettings); const createConfig: CreateSandboxConfig = { sessionId, + retireSandboxId: priorSandboxId, sandboxId: expectedSandboxId, repoOwner: session.repo_owner, repoName: session.repo_name, @@ -819,6 +836,7 @@ export class SandboxLifecycleManager let result: CreateSandboxResult; try { + this.recordPendingProviderReference(generation, sessionId); result = await this.provider.createSandbox(createConfig); } catch (error) { if (!selectedImage) throw error; @@ -854,6 +872,7 @@ export class SandboxLifecycleManager preserveProviderObjectId: false, shutdownPolicy: shutdownPolicyForLaunch("new", null), })); + this.recordPendingProviderReference(generation, sessionId); result = await this.provider.createSandbox({ ...createConfig, sandboxId: expectedSandboxId, @@ -895,6 +914,7 @@ export class SandboxLifecycleManager }); return; } + await this.handleRejectedStartupAllocation(error, generation); const errorMessage = error instanceof Error ? error.message : "Failed to spawn sandbox"; this.log.error("Sandbox spawn completed", { event: "sandbox.spawn", @@ -1170,6 +1190,10 @@ export class SandboxLifecycleManager this.storage.setLastSpawnError(null, null); const now = Date.now(); + const priorSandbox = this.storage.getSandbox(); + const priorSandboxId = priorSandbox?.modal_sandbox_id ?? null; + // A fenced allocation must be retired before its durable identity is replaced. + if (priorSandbox?.fenced) await this.stopPriorProviderSandbox(true); const reserved = this.spawnGeneration(session, now); generation = reserved; const shutdownPolicy = shutdownPolicyForLaunch("existing", snapshotRuntimeVersion); @@ -1197,8 +1221,10 @@ export class SandboxLifecycleManager const sandboxSettings = this.parseSandboxSettings(session); const timeoutSeconds = this.resolveSandboxTimeoutSeconds(sandboxSettings); this.shutdown.markRecoveryInvoked(generation); + this.recordPendingProviderReference(generation, session.session_name || session.id); const result = await this.provider.restoreFromSnapshot({ snapshotImageId, + retireSandboxId: priorSandboxId, sessionId: session.session_name || session.id, sandboxId: expectedSandboxId, sandboxAuthToken, @@ -1288,6 +1314,7 @@ export class SandboxLifecycleManager }); return; } + await this.handleRejectedStartupAllocation(error, generation); const errorMessage = error instanceof Error ? error.message : "Failed to restore sandbox"; this.log.error("Sandbox restore completed", { event: "sandbox.restore", @@ -1466,9 +1493,14 @@ export class SandboxLifecycleManager */ async triggerSnapshot(reason: string): Promise { if (this.shutdown.isHolding()) return; - // A Vercel snapshot stops the source. It requires the same preparation - // and replacement ordering as a final snapshot, even after a prompt. - if (this.provider.capabilities.snapshotStopsSandbox) { + // Some providers require terminal shutdown before capturing an ordinary + // checkpoint. The source may be retired by the provider or after the + // control plane commits its capture receipt. + if (this.provider.capabilities.snapshotRequiresShutdown) { + // Saving after every turn would stop the sandbox after every turn. It + // keeps running instead; inactivity, lifetime expiry and failures save + // it on the way down. + if (reason === "execution_complete") return; const ownership = await this.shutdown.requestShutdown(reason); if (ownership !== "unmanaged") return; } @@ -1520,7 +1552,7 @@ export class SandboxLifecycleManager /** * Stop a sandbox that is about to be replaced before its provider handle is cleared. */ - private async stopPriorProviderSandbox(): Promise { + private async stopPriorProviderSandbox(requireConfirmation = false): Promise { const providerObjectId = this.storage.getSandbox()?.modal_object_id; if (!providerObjectId) { return; @@ -1546,6 +1578,7 @@ export class SandboxLifecycleManager ]); this.storage.updateSandboxModalObjectId(null); } catch (error) { + if (requireConfirmation) throw error; this.storage.updateSandboxModalObjectId(null); this.log.warn("Provider stop failed before sandbox replacement", { provider_object_id: providerObjectId, @@ -1774,6 +1807,13 @@ export class SandboxLifecycleManager threshold_ms: this.config.heartbeat.timeoutMs, sandbox_status: ctx.sandbox.status, }); + if (!isBooting && this.provider.capabilities.snapshotRequiresShutdown) { + // These providers save only on the way down, and a runtime that stopped + // heartbeating cannot take part in a graceful drain. The coordinator + // captures the source without it, then stops it. + const ownership = await this.shutdown.requestShutdown("heartbeat_timeout", "emergency"); + if (ownership !== "unmanaged") return "no_action"; + } this.storage.updateSandboxStatus("stale"); // A bridge that connected and then died mid-boot is a boot failure // like any other; the termination re-drives the queue, and the breaker @@ -2126,7 +2166,15 @@ export class SandboxLifecycleManager } } - handleShutdownAlarm(): Promise<"continue" | "hold_watchdogs"> { + async handleShutdownAlarm(): Promise<"continue" | "hold_watchdogs"> { + const rejected = this.storage.getSandbox(); + if (rejected?.startup_rejected && rejected.modal_object_id) { + await this.attemptRejectedStartupCleanup( + { sandboxId: rejected.modal_sandbox_id, createdAt: rejected.created_at }, + rejected.modal_object_id + ); + return "hold_watchdogs"; + } return this.shutdown.handleAlarm(); } @@ -2138,13 +2186,39 @@ export class SandboxLifecycleManager return this.shutdown.snapshot(); } - /** Session cancellation preserves its existing shutdown-before-status policy. */ - cancelSandbox(): void { - if (!shouldStopSandboxOnSessionCancel(this.storage.getSandbox()?.status)) return; + /** + * Session cancellation preserves its existing shutdown-before-status policy, + * then destroys the sandbox at the provider: a runtime that is not connected + * never receives the shutdown, and a cancelled session never resumes a + * preserved one. + */ + async cancelSandbox(): Promise { + const sandbox = this.storage.getSandbox(); + if (!shouldStopSandboxOnSessionCancel(sandbox?.status)) return; if (this.wsManager.getSandboxWebSocket()) { this.wsManager.sendToSandbox({ type: "shutdown" }); } this.storage.updateSandboxStatus("stopped"); + if (sandbox?.modal_object_id && this.canStopProviderSandbox()) { + await this.stopProviderSandboxSafely({ + reason: "session_cancelled", + intent: "destroy", + providerObjectId: sandbox.modal_object_id, + failureMessage: "Provider stop failed after session cancel", + }); + } + } + + /** + * An archived session's sandbox is saved and stopped now: its runtime may + * hold work no save covers yet, and archive refuses its reconnects. + */ + async preserveForArchive(): Promise { + await this.shutdown.requestShutdown("session_archived"); + } + + onRefusedReconnect(): "retry" | "exit" { + return this.shutdown.onRefusedReconnect(); } /** Update last activity timestamp. */ @@ -2278,6 +2352,69 @@ export class SandboxLifecycleManager await this.storage.updateSandboxAccess("ttyd", url, token); } + private recordPendingProviderReference(generation: SandboxGeneration, sessionId: string): void { + if (!generation.sandboxId) throw new SpawnSupersededError(); + const reference = this.provider.pendingSandboxReference?.(sessionId, generation.sandboxId); + if (!reference) return; + const row = this.storage.getSandbox(); + if ( + row?.modal_sandbox_id !== generation.sandboxId || + row.created_at !== generation.createdAt || + row.fenced + ) { + throw new SpawnSupersededError(); + } + this.storage.updateSandboxModalObjectId(reference); + } + + private async handleRejectedStartupAllocation( + error: unknown, + generation: SandboxGeneration | null + ): Promise { + if (!(error instanceof SandboxLaunchRejectedError) || !generation) return; + // A rejected launch may already have connected. Fence its credentials and retain + // its provider ID before termination so a failed stop or DO restart cannot + // accept the allocation or lose the cleanup obligation. + const rejection = this.storage.rejectProviderStartup(generation, error.providerObjectId); + if (rejection === "superseded") { + await this.destroyLateProviderResult(error.providerObjectId ?? undefined); + return; + } + this.wsManager.detachSandboxWebSocket(1008, "Provider allocation rejected"); + this.clearSandboxAccessState(); + if (rejection === "failed") { + this.broadcaster.broadcast({ type: "sandbox_status", status: "failed" }); + this.reportSandboxError(error.message); + this.recordSpawnFailure(Date.now(), generation.createdAt); + } + if (error.providerObjectId) + await this.attemptRejectedStartupCleanup(generation, error.providerObjectId); + } + + async rearmRejectedStartupCleanupAlarm(): Promise { + const row = this.storage.getSandbox(); + if (row?.startup_rejected && row.modal_object_id) { + await this.alarmScheduler.schedule(Date.now() + REJECTED_ALLOCATION_CLEANUP_RETRY_MS); + } + } + + private async attemptRejectedStartupCleanup( + generation: SandboxGeneration, + providerObjectId: string + ): Promise { + // Persist the next attempt before provider I/O so an eviction cannot lose cleanup. + await this.rearmRejectedStartupCleanupAlarm(); + if (!(await this.destroyLateProviderResult(providerObjectId))) return; + const row = this.storage.getSandbox(); + if ( + row?.modal_sandbox_id === generation.sandboxId && + row.created_at === generation.createdAt && + row.modal_object_id === providerObjectId + ) { + this.storage.updateSandboxModalObjectId(null); + } + } + private async claimProviderStartup( generation: SandboxGeneration, providerObjectId: string | undefined, @@ -2325,8 +2462,8 @@ export class SandboxLifecycleManager return true; } - private async destroyLateProviderResult(providerObjectId: string | undefined): Promise { - if (!providerObjectId || !this.canStopProviderSandbox()) return; + private async destroyLateProviderResult(providerObjectId: string | undefined): Promise { + if (!providerObjectId || !this.canStopProviderSandbox()) return false; const controller = new AbortController(); let timeoutId: ReturnType | undefined; try { @@ -2345,11 +2482,13 @@ export class SandboxLifecycleManager ), timeout, ]); + return true; } catch (error) { this.log.warn("Failed to destroy superseded provider sandbox", { provider_object_id: providerObjectId, error: error instanceof Error ? error.message : String(error), }); + return false; } finally { if (timeoutId !== undefined) clearTimeout(timeoutId); } diff --git a/packages/control-plane/src/sandbox/lifecycle/ports.ts b/packages/control-plane/src/sandbox/lifecycle/ports.ts index 3d49f34401..b625cce996 100644 --- a/packages/control-plane/src/sandbox/lifecycle/ports.ts +++ b/packages/control-plane/src/sandbox/lifecycle/ports.ts @@ -45,9 +45,13 @@ export interface SandboxReadiness { onRuntimeReady(timestamp: number, harness?: string, protocolVersion?: 1): boolean; } -/** Applies sandbox cancellation after session work has been cancelled. */ +/** + * Applies sandbox cancellation after session work has been cancelled, and + * saves the sandbox of a session that is archived. + */ export interface SandboxCancellation { - cancelSandbox(): void; + cancelSandbox(): Promise; + preserveForArchive(): Promise; } /** Transport attachment reports facts without granting ordinary command readiness. */ @@ -56,6 +60,11 @@ export interface SandboxAttachment { isProviderStartupPending(): boolean; onSandboxConnected(): void; onSandboxSocketAttached(generation: SandboxGeneration): void; + /** + * A runtime refused at reconnect normally exits, which ends its sandbox. + * "retry" keeps it up because a capture still needs that sandbox. + */ + onRefusedReconnect(): "retry" | "exit"; } /** The lifecycle result consumed by the existing alarm coordinator. */ diff --git a/packages/control-plane/src/sandbox/lifecycle/rejected-allocation.test.ts b/packages/control-plane/src/sandbox/lifecycle/rejected-allocation.test.ts new file mode 100644 index 0000000000..954a32a658 --- /dev/null +++ b/packages/control-plane/src/sandbox/lifecycle/rejected-allocation.test.ts @@ -0,0 +1,188 @@ +import { describe, expect, it, vi } from "vitest"; +import { COMPATIBLE_RUNTIME_VERSION } from "../../image-builds/test-helpers"; +import { createAlarmHandler } from "../../session/alarm/handler"; +import { SandboxLaunchRejectedError } from "../provider"; +import { ModalSandboxProvider } from "../providers/modal-provider"; +import type { ModalClient } from "../client"; +import { createAlarmFixture, createMockProvider, createMockSandbox } from "./test-helpers"; + +describe("rejected provider allocation", () => { + it.each(["stopped", "stale"] as const)( + "preserves %s when a cancelled launch is later rejected", + async (status) => { + const sandbox = createMockSandbox({ status: "pending", modal_object_id: null }); + const provider = createMockProvider({ + capabilities: { supportsExplicitStop: true }, + stopSandbox: async () => { + throw new Error("unavailable"); + }, + createSandbox: async () => { + sandbox.status = status; + throw new SandboxLaunchRejectedError("incompatible", "sb-rejected"); + }, + }); + const fixture = createAlarmFixture(sandbox, provider); + await fixture.manager.spawnSandbox(); + expect(sandbox.status).toBe(status); + expect(sandbox.fenced).toBe(1); + expect(sandbox.modal_object_id).toBe("sb-rejected"); + expect(fixture.broadcaster.broadcast).not.toHaveBeenCalledWith({ + type: "sandbox_status", + status: "failed", + }); + expect(fixture.storage.incrementCircuitBreakerFailure).not.toHaveBeenCalled(); + } + ); + it("retains a pre-launch VM reference when the response is lost after bridge connection", async () => { + const sandbox = createMockSandbox({ status: "pending", modal_object_id: null }); + const client = { + createSandbox: vi.fn(async () => { + expect(sandbox.modal_object_id).toContain("modal-vm-session:"); + sandbox.status = "ready"; + vi.mocked(fixture.wsManager.getSandboxWebSocket).mockReturnValue({} as WebSocket); + throw new Error("response lost"); + }), + }; + const provider = new ModalSandboxProvider(client as unknown as ModalClient, "modal-vm"); + const fixture = createAlarmFixture(sandbox, provider); + await fixture.manager.spawnSandbox(); + expect(sandbox.status).toBe("ready"); + expect(sandbox.modal_object_id).toContain(sandbox.modal_sandbox_id); + const restarted = createAlarmFixture(sandbox, provider); + expect(restarted.storage.getSandbox()?.modal_object_id).toBe(sandbox.modal_object_id); + }); + it("fences an early connected generation before waiting for mismatch cleanup", async () => { + const sandbox = createMockSandbox({ status: "pending", modal_object_id: null }); + let finishStop!: () => void; + const client = { + createSandbox: vi.fn(async () => { + sandbox.status = "ready"; + vi.mocked(fixture.wsManager.getSandboxWebSocket).mockReturnValue({} as WebSocket); + return { + sandboxId: sandbox.modal_sandbox_id, + modalObjectId: "sb-rejected", + sandboxBackend: "modal", + createdAt: 1, + }; + }), + stopSandbox: vi.fn( + () => + new Promise((resolve) => { + finishStop = resolve; + }) + ), + }; + const fixture = createAlarmFixture( + sandbox, + new ModalSandboxProvider(client as unknown as ModalClient, "modal-vm") + ); + const spawning = fixture.manager.spawnSandbox(); + try { + await vi.waitFor(() => expect(client.stopSandbox).toHaveBeenCalledOnce()); + expect(sandbox.fenced).toBe(1); + expect(sandbox.auth_token_hash).toBe(""); + expect(sandbox.modal_object_id).toBe("sb-rejected"); + expect(fixture.wsManager.detachSandboxWebSocket).toHaveBeenCalled(); + expect(fixture.manager.mayProcessQueuedWork()).toBe(false); + } finally { + finishStop(); + await spawning; + } + }); + it.each([ + [false, "sb-rejected"], + [true, "sb-rejected"], + ["expired", "sb-rejected"], + [false, null], + [true, null], + ["expired", null], + ] as const)( + "retains cleanup identity and fences the rejected generation (early bridge=%s, cleanup handle=%s)", + async (earlyBridge, cleanupHandle) => { + const sandbox = createMockSandbox({ status: "pending", modal_object_id: null }); + const provider = createMockProvider({ + capabilities: { supportsExplicitStop: true }, + stopSandbox: async () => { + throw new Error("retirement unavailable"); + }, + createSandbox: async () => { + if (earlyBridge) { + sandbox.status = earlyBridge === "expired" ? "failed" : "connecting"; + if (earlyBridge === "expired") sandbox.fenced = 1; + vi.mocked(fixture.wsManager.getSandboxWebSocket).mockReturnValue({} as WebSocket); + } + throw new SandboxLaunchRejectedError("incompatible allocation", cleanupHandle); + }, + }); + const fixture = createAlarmFixture(sandbox, provider); + await fixture.manager.spawnSandbox(); + expect(sandbox.status).toBe("failed"); + expect(sandbox.fenced).toBe(1); + expect(sandbox.auth_token_hash).toBe(""); + expect(sandbox.modal_object_id).toBe(cleanupHandle); + expect(fixture.wsManager.detachSandboxWebSocket).toHaveBeenCalled(); + } + ); + it("retains rejected cleanup responsibility across restart and failed retirement", async () => { + const sandbox = createMockSandbox({ status: "pending", modal_object_id: null }); + const provider = createMockProvider({ + capabilities: { supportsExplicitStop: true }, + createSandbox: vi.fn(async () => { + throw new SandboxLaunchRejectedError("incompatible", "sb-rejected"); + }), + stopSandbox: vi.fn(async () => { + throw new Error("provider unavailable"); + }), + }); + await createAlarmFixture(sandbox, provider).manager.spawnSandbox(); + const rejectedGeneration = sandbox.modal_sandbox_id; + const restarted = createAlarmFixture(sandbox, provider); + await restarted.manager.spawnSandbox(); + await restarted.manager.spawnSandbox(); + expect(sandbox.modal_object_id).toBe("sb-rejected"); + expect(sandbox.modal_sandbox_id).toBe(rejectedGeneration); + expect(sandbox.fenced).toBe(1); + expect(provider.createSandbox).toHaveBeenCalledTimes(1); + }); + it.each(["create", "restore"] as const)( + "rearms %s cleanup through the assembled alarm handler even under a shutdown hold", + async (launch) => { + const sandbox = createMockSandbox({ status: "pending", modal_object_id: null }); + const stopSandbox = vi.fn().mockRejectedValue(new Error("provider unavailable")); + const provider = createMockProvider({ + capabilities: { supportsExplicitStop: true }, + stopSandbox, + createSandbox: async () => { + throw new SandboxLaunchRejectedError("incompatible", "sb-rejected"); + }, + }); + const first = createAlarmFixture(sandbox, provider); + if (launch === "restore") { + vi.mocked(first.shutdown.startupDecision).mockReturnValue({ + kind: "restore_snapshot", + snapshotId: "im-existing", + runtimeVersion: COMPATIBLE_RUNTIME_VERSION, + }); + provider.restoreFromSnapshot = provider.createSandbox as never; + } + await first.manager.spawnSandbox(); + const restarted = createAlarmFixture(sandbox, provider); + vi.spyOn(restarted.shutdown, "handleAlarm").mockImplementation( + async () => "hold_watchdogs" as never + ); + const handler = createAlarmHandler({ + preserveBeforeWatchdogs: () => restarted.manager.handleShutdownAlarm(), + lifecycleManager: restarted.manager, + terminalMessageProjection: { flushPending: vi.fn(async () => {}) }, + } as never); + await restarted.manager.rearmRejectedStartupCleanupAlarm(); + expect(restarted.alarmScheduler.schedule).toHaveBeenCalled(); + await handler.handle(); + expect(sandbox.modal_object_id).toBe("sb-rejected"); + stopSandbox.mockResolvedValue({ success: true }); + await handler.handle(); + expect(sandbox.modal_object_id).toBeNull(); + expect(sandbox.fenced).toBe(1); + } + ); +}); diff --git a/packages/control-plane/src/sandbox/lifecycle/test-helpers.ts b/packages/control-plane/src/sandbox/lifecycle/test-helpers.ts index 710065ef85..290d85d072 100644 --- a/packages/control-plane/src/sandbox/lifecycle/test-helpers.ts +++ b/packages/control-plane/src/sandbox/lifecycle/test-helpers.ts @@ -97,6 +97,7 @@ export function createMockSandbox( boot_phase: null, boot_seq: null, fenced: 0, + startup_rejected: 0, created_at: Date.now() - 60000, spawn_failure_count: 0, last_spawn_failure: 0, @@ -162,6 +163,23 @@ export function createMockStorage( return true; } ), + rejectProviderStartup: vi.fn((generation, providerObjectId) => { + if ( + !sandbox || + sandbox.modal_sandbox_id !== generation.sandboxId || + sandbox.created_at !== generation.createdAt + ) + return "superseded"; + sandbox.modal_object_id = providerObjectId; + const failed = ["spawning", "connecting", "ready"].includes(sandbox.status); + if (failed) sandbox.status = "failed"; + sandbox.fenced = 1; + sandbox.startup_rejected = 1; + sandbox.auth_token_hash = ""; + sandbox.auth_token = null; + sandbox.active_socket_id = ""; + return failed ? "failed" : "retained"; + }), commitProviderStartup: vi.fn((generation, providerObjectId, allowFailedSelfHeal) => { calls.push("commitProviderStartup"); if ( @@ -184,6 +202,7 @@ export function createMockStorage( calls.push("updateSandboxForSpawn"); if (sandbox) { sandbox.status = data.status; + sandbox.startup_rejected = 0; sandbox.created_at = data.createdAt; sandbox.auth_token_hash = ""; sandbox.auth_token = null; @@ -435,6 +454,7 @@ export function createUnmanagedShutdown() { markRecoveryInvoked: vi.fn(), recordProviderStartup: vi.fn(async () => {}), isHolding: vi.fn(() => false), + onRefusedReconnect: vi.fn(() => "exit" as const), requestShutdown: vi.fn(async () => "unmanaged"), captureCheckpoint: vi.fn(async () => ({ outcome: "saved", @@ -495,6 +515,7 @@ export function createCheckpointShutdown( : Promise.resolve("unmanaged"), isHolding: () => coordinator.isHolding(), admissionDecision: () => coordinator.admissionDecision(), + onRefusedReconnect: () => coordinator.onRefusedReconnect(), }; } diff --git a/packages/control-plane/src/sandbox/provider-factory.ts b/packages/control-plane/src/sandbox/provider-factory.ts index 271aaa0dbc..90615f3e1a 100644 --- a/packages/control-plane/src/sandbox/provider-factory.ts +++ b/packages/control-plane/src/sandbox/provider-factory.ts @@ -21,10 +21,10 @@ import { createVercelProvider, type VercelSandboxProvider } from "./providers/ve import { resolveScmProviderFromEnv } from "../source-control"; import type { Env } from "../types"; -function createModalProviderFromEnv(env: Env): ModalSandboxProvider { +function createModalProviderFromEnv(env: Env, backend: "modal" | "modal-vm"): ModalSandboxProvider { if (!env.MODAL_API_SECRET || !env.MODAL_WORKSPACE) { throw new Error( - "MODAL_API_SECRET and MODAL_WORKSPACE are required when SANDBOX_PROVIDER=modal" + `MODAL_API_SECRET and MODAL_WORKSPACE are required when SANDBOX_PROVIDER=${backend}` ); } @@ -35,7 +35,7 @@ function createModalProviderFromEnv(env: Env): ModalSandboxProvider { env.MODAL_API_URL ); - return createModalProvider(client); + return createModalProvider(client, backend); } function createVercelProviderFromEnv(env: Env): VercelSandboxProvider { @@ -170,7 +170,10 @@ function createE2BProviderFromEnv(env: Env): E2BSandboxProvider { export function createSandboxProviderFromEnv(env: Env, backend: "daytona"): DaytonaSandboxProvider; export function createSandboxProviderFromEnv(env: Env, backend: "e2b"): E2BSandboxProvider; -export function createSandboxProviderFromEnv(env: Env, backend: "modal"): ModalSandboxProvider; +export function createSandboxProviderFromEnv( + env: Env, + backend: "modal" | "modal-vm" +): ModalSandboxProvider; export function createSandboxProviderFromEnv(env: Env, backend: "vercel"): VercelSandboxProvider; export function createSandboxProviderFromEnv( env: Env, @@ -199,7 +202,8 @@ export function createSandboxProviderFromEnv( case "e2b": return createE2BProviderFromEnv(env); case "modal": - return createModalProviderFromEnv(env); + case "modal-vm": + return createModalProviderFromEnv(env, backend); } } diff --git a/packages/control-plane/src/sandbox/provider.ts b/packages/control-plane/src/sandbox/provider.ts index 2e7317a1e5..6093b8340d 100644 --- a/packages/control-plane/src/sandbox/provider.ts +++ b/packages/control-plane/src/sandbox/provider.ts @@ -62,8 +62,8 @@ export interface SandboxProviderCapabilities { supportsPersistentResume?: boolean; /** Whether the provider can stop a sandbox explicitly via API */ supportsExplicitStop?: boolean; - /** Whether taking a snapshot also stops the source sandbox. */ - snapshotStopsSandbox?: boolean; + /** An ordinary checkpoint must enter the terminal shutdown flow first. */ + snapshotRequiresShutdown?: boolean; } export type SandboxLifetime = @@ -145,6 +145,8 @@ export interface CreateSandboxConfig { mcpServers?: McpServerConfig[]; /** Sandbox settings (tunnel ports, etc.) resolved from integration settings */ sandboxSettings?: SandboxSettings; + /** Previous logical allocation identity, used by providers supporting ambiguous-create recovery. */ + retireSandboxId?: string | null; /** * Ordered member list for multi-repo sessions. Only set when the session * has more than one member — single-repo sessions keep the scalar @@ -235,6 +237,8 @@ export interface RestoreConfig { agentSlackNotifyEnabled?: boolean; /** Sandbox settings (tunnel ports, etc.) resolved from integration settings */ sandboxSettings?: SandboxSettings; + /** Previous logical allocation identity, used by providers supporting ambiguous-create recovery. */ + retireSandboxId?: string | null; /** Multi-repo member list — see CreateSandboxConfig. */ repositories?: SessionRepositoryInfo[]; } @@ -303,6 +307,8 @@ export interface SnapshotResult { error?: string; /** True when snapshot creation itself stopped the source sandbox. */ sourceStopped?: boolean; + /** Immutable source ID to retire after committing the snapshot receipt. */ + sourceObjectId?: string; } /** @@ -491,6 +497,18 @@ export class SandboxProviderError extends Error { } } +/** A rejected execution; a non-null handle remains a cleanup obligation. */ +export class SandboxLaunchRejectedError extends SandboxProviderError { + constructor( + message: string, + readonly providerObjectId: string | null, + cause?: Error + ) { + super(message, "permanent", cause); + this.name = "SandboxLaunchRejectedError"; + } +} + /** The provider confirmed that the selected prebuilt artifact cannot be restored. */ export class PrebuiltImageUnavailableError extends SandboxProviderError { constructor(message: string, cause?: Error) { @@ -550,6 +568,11 @@ export interface SandboxProvider { /** Provider capabilities */ readonly capabilities: SandboxProviderCapabilities; + /** Optional opaque reference usable for snapshot/stop even if the launch response is lost. + * Persisted before launch; this is not evidence that startup succeeded. + */ + pendingSandboxReference?(sessionId: string, sandboxId: string): string | undefined; + /** * Create a new sandbox. * diff --git a/packages/control-plane/src/sandbox/providers/modal-backends.test.ts b/packages/control-plane/src/sandbox/providers/modal-backends.test.ts new file mode 100644 index 0000000000..78e621b561 --- /dev/null +++ b/packages/control-plane/src/sandbox/providers/modal-backends.test.ts @@ -0,0 +1,277 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { createModalClient, type ModalClient } from "../client"; +import { SandboxLaunchRejectedError } from "../provider"; +import { ModalSandboxProvider } from "./modal-provider"; +import { resolveSandboxDashboardUrl } from "../../session/sandbox-access"; + +const config = { + sessionId: "session-1", + sandboxId: "sandbox-1", + repoOwner: null, + repoName: null, + controlPlaneUrl: "https://cp.test", + sandboxAuthToken: "token", + harness: "opencode" as const, + provider: "anthropic", + model: "model", + retireSandboxId: "prior-generation", +}; +const build = { + buildId: "build-1", + scopeKind: "repo" as const, + scopeId: "acme/repo", + repositories: [{ repoOwner: "acme", repoName: "repo", baseBranch: "main" }], + callbackUrl: "https://cp.test/complete", + failureCallbackUrl: "https://cp.test/failed", + callbackToken: "token", + buildExecutionTimeoutSeconds: 60, + providerSessionTimeoutSeconds: 120, + correlation: { trace_id: "trace", request_id: "request" }, +}; + +function fixture(confirmation: unknown) { + const result = { + sandboxId: "sandbox-1", + modalObjectId: "sb-1", + createdAt: 1, + sandboxBackend: confirmation, + }; + const client = { + createSandbox: vi.fn().mockResolvedValue(result), + restoreSandbox: vi.fn().mockResolvedValue(result), + stopSandbox: vi.fn().mockResolvedValue(undefined), + createImageBuildSandbox: vi + .fn() + .mockResolvedValue({ providerSessionId: "sb-1", sandboxBackend: confirmation }), + startImageBuildSandbox: vi.fn().mockResolvedValue(undefined), + snapshotSandbox: vi + .fn() + .mockResolvedValue({ imageId: "im-1", sourceStopped: false, sourceObjectId: "sb-1" }), + }; + return { + client, + provider: new ModalSandboxProvider(client as unknown as ModalClient, "modal-vm"), + }; +} + +afterEach(() => vi.unstubAllGlobals()); + +describe("distinct Modal backend identities", () => { + it("retries a lost VM capture response while retaining the source", async () => { + const { client, provider } = fixture("modal-vm"); + client.snapshotSandbox.mockRejectedValueOnce(new Error("response lost")); + await expect( + provider.takeSnapshot({ + providerObjectId: "sb-1", + sessionId: "session-1", + reason: "shutdown", + deadlineAtMs: Date.now() + 60_000, + }) + ).resolves.toMatchObject({ success: true, imageId: "im-1", sourceStopped: false }); + expect(client.snapshotSandbox).toHaveBeenCalledTimes(2); + expect(client.snapshotSandbox.mock.calls[0]).toEqual(client.snapshotSandbox.mock.calls[1]); + }); + it("selects the immutable backend on both launch paths without altering generic resources", async () => { + const { provider, client } = fixture("modal-vm"); + const settings = { cpuCores: 3, memoryMib: null }; + await provider.createSandbox({ ...config, sandboxSettings: settings }); + await provider.restoreFromSnapshot({ + ...config, + snapshotImageId: "im-1", + sandboxSettings: settings, + }); + for (const call of [client.createSandbox, client.restoreSandbox]) { + expect(call).toHaveBeenCalledWith( + expect.objectContaining({ + sandboxBackend: "modal-vm", + retireSandboxId: "prior-generation", + sandboxSettings: settings, + }), + undefined + ); + } + expect(provider.name).toBe("modal-vm"); + expect(provider.capabilities.snapshotRequiresShutdown).toBe(true); + }); + + it.each([undefined, null, false, "modal", "future-backend", { unexpected: true }])( + "returns rejected create/restore handles before cleanup with confirmation %j", + async (value) => { + const { provider, client } = fixture(value); + await expect(provider.createSandbox(config)).rejects.toThrow("did not confirm"); + await expect( + provider.restoreFromSnapshot({ ...config, snapshotImageId: "im-1" }) + ).rejects.toThrow("did not confirm"); + expect(client.stopSandbox).not.toHaveBeenCalled(); + } + ); + + it("accepts older standard Modal responses without a backend echo", async () => { + const { client } = fixture(undefined); + const provider = new ModalSandboxProvider(client as unknown as ModalClient, "modal"); + await expect(provider.createSandbox(config)).resolves.toMatchObject({ + providerObjectId: "sb-1", + }); + await expect( + provider.restoreFromSnapshot({ ...config, snapshotImageId: "im-1" }) + ).resolves.toMatchObject({ providerObjectId: "sb-1" }); + const bind = vi.fn().mockResolvedValue(undefined); + await provider.triggerImageBuild({ ...build, onProviderSessionCreated: bind }); + expect(bind).toHaveBeenCalledWith("sb-1"); + expect(client.startImageBuildSandbox).toHaveBeenCalledOnce(); + }); + + it("carries the rejected session allocation ID for lifecycle-owned cleanup", async () => { + const { provider, client } = fixture("modal"); + client.stopSandbox.mockRejectedValue(new Error("unreachable")); + await expect(provider.createSandbox(config)).rejects.toMatchObject({ + name: "SandboxLaunchRejectedError", + providerObjectId: "sb-1", + }); + await expect( + provider.restoreFromSnapshot({ ...config, snapshotImageId: "im-1" }) + ).rejects.toBeInstanceOf(SandboxLaunchRejectedError); + }); + + it.each([undefined, "modal", 42, { bad: true }])( + "binds rejected build handles for cleanup but never starts them (%j)", + async (value) => { + const { provider, client } = fixture(value); + const bind = vi.fn().mockResolvedValue(undefined); + await expect( + provider.triggerImageBuild({ ...build, onProviderSessionCreated: bind }) + ).rejects.toThrow("did not confirm"); + expect(bind).toHaveBeenCalledWith("sb-1"); + expect(client.startImageBuildSandbox).not.toHaveBeenCalled(); + } + ); + + it("starts a confirmed build only after binding", async () => { + const { provider, client } = fixture("modal-vm"); + let bound = false; + client.startImageBuildSandbox.mockImplementation(async () => expect(bound).toBe(true)); + await provider.triggerImageBuild({ + ...build, + onProviderSessionCreated: async () => { + bound = true; + }, + }); + expect(client.startImageBuildSandbox).toHaveBeenCalledOnce(); + }); + + it("preserves an allocation handle when wire confirmation is malformed", async () => { + vi.stubGlobal( + "fetch", + vi.fn().mockResolvedValue( + Response.json({ + success: true, + data: { provider_session_id: "sb-1", sandbox_backend: { invalid: true } }, + }) + ) + ); + const client = createModalClient("test-secret", "workspace"); + const result = await client.createImageBuildSandbox({ ...build, sandboxBackend: "modal-vm" }); + expect(result).toMatchObject({ providerSessionId: "sb-1", sandboxBackend: { invalid: true } }); + }); + + it("serializes backend and predecessor identity on create and restore HTTP requests", async () => { + const fetchMock = vi.fn().mockResolvedValue( + Response.json({ + success: true, + data: { + sandbox_id: "sandbox-1", + modal_object_id: "sb-1", + created_at: 1, + sandbox_backend: "modal-vm", + }, + }) + ); + vi.stubGlobal("fetch", fetchMock); + const client = createModalClient("test-secret", "workspace"); + await client.createSandbox({ ...config, sandboxBackend: "modal-vm" }); + fetchMock.mockResolvedValue( + Response.json({ + success: true, + data: { sandbox_id: "sandbox-1", modal_object_id: "sb-1", sandbox_backend: "modal-vm" }, + }) + ); + await client.restoreSandbox({ ...config, snapshotImageId: "im-1", sandboxBackend: "modal-vm" }); + for (const [, init] of fetchMock.mock.calls) { + expect(JSON.parse(init.body)).toMatchObject({ + sandbox_backend: "modal-vm", + retire_sandbox_id: "prior-generation", + }); + } + }); + + it("leaves VM retirement to the control plane after capture", async () => { + const { provider, client } = fixture("modal-vm"); + const input = { providerObjectId: "sb-1", sessionId: "session-1", reason: "checkpoint" }; + await expect(provider.takeSnapshot(input)).resolves.toMatchObject({ + success: true, + sourceStopped: false, + sourceObjectId: "sb-1", + }); + expect(client.stopSandbox).not.toHaveBeenCalled(); + }); + + it("uses a separate VM capture endpoint so older deployments cannot stop the source", async () => { + const fetchMock = vi.fn().mockImplementation(async () => + Response.json({ + success: true, + data: { image_id: "im-1", source_stopped: false, source_id: "sb-1", sandbox_id: "sb-1" }, + }) + ); + vi.stubGlobal("fetch", fetchMock); + const client = createModalClient("secret", "acme"); + await client.snapshotSandbox({ + providerObjectId: "sb-1", + sessionId: "session-1", + sandboxBackend: "modal-vm", + }); + await client.snapshotSandbox({ + providerObjectId: "sb-2", + sessionId: "session-2", + sandboxBackend: "modal", + }); + expect(fetchMock.mock.calls[0][0]).toContain("api-snapshot-vm-sandbox"); + expect(fetchMock.mock.calls[1][0]).toContain("api-snapshot-sandbox"); + }); + + it("holds a VM capture without an immutable source ID", async () => { + const { provider, client } = fixture("modal-vm"); + client.snapshotSandbox.mockResolvedValue({ imageId: "im-1", sourceStopped: false }); + await expect( + provider.takeSnapshot({ + providerObjectId: "pending-ref", + sessionId: "session-1", + reason: "checkpoint", + }) + ).rejects.toThrow("did not confirm its source ID"); + }); + + it("holds a VM capture that does not confirm source retention", async () => { + const { provider, client } = fixture("modal-vm"); + client.snapshotSandbox.mockResolvedValue({ + imageId: "im-1", + sourceStopped: true, + sourceObjectId: "sb-1", + }); + await expect( + provider.takeSnapshot({ + providerObjectId: "sb-1", + sessionId: "session-1", + reason: "checkpoint", + }) + ).rejects.toThrow("did not confirm source retention"); + }); + + it.each(["modal", "modal-vm"])("keeps dashboard links for %s", (backend) => { + expect( + resolveSandboxDashboardUrl( + { sandboxProvider: backend, modalWorkspace: "acme", modalEnvironment: "main" }, + "sb-1" + ) + ).toContain("sandboxId=sb-1"); + }); +}); diff --git a/packages/control-plane/src/sandbox/providers/modal-provider.test.ts b/packages/control-plane/src/sandbox/providers/modal-provider.test.ts index b7efd04ffd..0d4ba3bc2f 100644 --- a/packages/control-plane/src/sandbox/providers/modal-provider.test.ts +++ b/packages/control-plane/src/sandbox/providers/modal-provider.test.ts @@ -97,7 +97,7 @@ describe("ModalSandboxProvider", () => { try { vi.setSystemTime(new Date("2030-01-01T00:00:00.000Z")); const client = createMockModalClient(); - const provider = new ModalSandboxProvider(client); + const provider = new ModalSandboxProvider(client, "modal"); const created = await provider.createSandbox({ ...testConfig, timeoutSeconds: 1200 }); expect(created.lifetime).toEqual({ kind: "finite", @@ -135,7 +135,7 @@ describe("ModalSandboxProvider", () => { describe("capabilities", () => { it("reports correct capabilities", () => { const client = createMockModalClient(); - const provider = new ModalSandboxProvider(client); + const provider = new ModalSandboxProvider(client, "modal"); expect(provider.name).toBe("modal"); expect(provider.capabilities.supportsSnapshots).toBe(true); @@ -151,7 +151,7 @@ describe("ModalSandboxProvider", () => { throw new Error("fetch failed"); }), }); - const provider = new ModalSandboxProvider(client); + const provider = new ModalSandboxProvider(client, "modal"); await expect(provider.createSandbox(testConfig)).rejects.toThrow(SandboxProviderError); try { @@ -168,7 +168,7 @@ describe("ModalSandboxProvider", () => { throw new Error("connect ETIMEDOUT 192.168.1.1:443"); }), }); - const provider = new ModalSandboxProvider(client); + const provider = new ModalSandboxProvider(client, "modal"); try { await provider.createSandbox(testConfig); @@ -184,7 +184,7 @@ describe("ModalSandboxProvider", () => { throw new Error("read ECONNRESET"); }), }); - const provider = new ModalSandboxProvider(client); + const provider = new ModalSandboxProvider(client, "modal"); try { await provider.createSandbox(testConfig); @@ -200,7 +200,7 @@ describe("ModalSandboxProvider", () => { throw new Error("connect ECONNREFUSED 127.0.0.1:3000"); }), }); - const provider = new ModalSandboxProvider(client); + const provider = new ModalSandboxProvider(client, "modal"); try { await provider.createSandbox(testConfig); @@ -216,7 +216,7 @@ describe("ModalSandboxProvider", () => { throw new Error("Network request failed"); }), }); - const provider = new ModalSandboxProvider(client); + const provider = new ModalSandboxProvider(client, "modal"); try { await provider.createSandbox(testConfig); @@ -232,7 +232,7 @@ describe("ModalSandboxProvider", () => { throw new RequestDeadlineError("Modal", "createSandbox", 30_000); }), }); - const provider = new ModalSandboxProvider(client); + const provider = new ModalSandboxProvider(client, "modal"); try { await provider.createSandbox(testConfig); @@ -248,7 +248,7 @@ describe("ModalSandboxProvider", () => { throw new Error("Modal API error: 502 Bad Gateway"); }), }); - const provider = new ModalSandboxProvider(client); + const provider = new ModalSandboxProvider(client, "modal"); try { await provider.createSandbox(testConfig); @@ -264,7 +264,7 @@ describe("ModalSandboxProvider", () => { throw new Error("Modal API error: 503 Service Unavailable"); }), }); - const provider = new ModalSandboxProvider(client); + const provider = new ModalSandboxProvider(client, "modal"); try { await provider.createSandbox(testConfig); @@ -280,7 +280,7 @@ describe("ModalSandboxProvider", () => { throw new Error("Modal API error: 504 Gateway Timeout"); }), }); - const provider = new ModalSandboxProvider(client); + const provider = new ModalSandboxProvider(client, "modal"); try { await provider.createSandbox(testConfig); @@ -296,7 +296,7 @@ describe("ModalSandboxProvider", () => { throw new Error("upstream bad gateway error"); }), }); - const provider = new ModalSandboxProvider(client); + const provider = new ModalSandboxProvider(client, "modal"); try { await provider.createSandbox(testConfig); @@ -312,7 +312,7 @@ describe("ModalSandboxProvider", () => { throw new Error("service unavailable, try again later"); }), }); - const provider = new ModalSandboxProvider(client); + const provider = new ModalSandboxProvider(client, "modal"); try { await provider.createSandbox(testConfig); @@ -328,7 +328,7 @@ describe("ModalSandboxProvider", () => { throw new Error("gateway timeout while waiting for upstream"); }), }); - const provider = new ModalSandboxProvider(client); + const provider = new ModalSandboxProvider(client, "modal"); try { await provider.createSandbox(testConfig); @@ -346,7 +346,7 @@ describe("ModalSandboxProvider", () => { throw new Error("Modal API error: 401 Unauthorized"); }), }); - const provider = new ModalSandboxProvider(client); + const provider = new ModalSandboxProvider(client, "modal"); try { await provider.createSandbox(testConfig); @@ -362,7 +362,7 @@ describe("ModalSandboxProvider", () => { throw new Error("Modal API error: 403 Forbidden"); }), }); - const provider = new ModalSandboxProvider(client); + const provider = new ModalSandboxProvider(client, "modal"); try { await provider.createSandbox(testConfig); @@ -378,7 +378,7 @@ describe("ModalSandboxProvider", () => { throw new Error("Modal API error: 400 Bad Request - Invalid configuration"); }), }); - const provider = new ModalSandboxProvider(client); + const provider = new ModalSandboxProvider(client, "modal"); try { await provider.createSandbox(testConfig); @@ -394,7 +394,7 @@ describe("ModalSandboxProvider", () => { throw new Error("Modal API error: 422 Unprocessable Entity"); }), }); - const provider = new ModalSandboxProvider(client); + const provider = new ModalSandboxProvider(client, "modal"); try { await provider.createSandbox(testConfig); @@ -410,7 +410,7 @@ describe("ModalSandboxProvider", () => { throw new Error("Invalid repository configuration"); }), }); - const provider = new ModalSandboxProvider(client); + const provider = new ModalSandboxProvider(client, "modal"); try { await provider.createSandbox(testConfig); @@ -426,7 +426,7 @@ describe("ModalSandboxProvider", () => { throw new Error("Quota exceeded: maximum sandboxes reached"); }), }); - const provider = new ModalSandboxProvider(client); + const provider = new ModalSandboxProvider(client, "modal"); try { await provider.createSandbox(testConfig); @@ -442,7 +442,7 @@ describe("ModalSandboxProvider", () => { throw new Error("Something unexpected happened"); }), }); - const provider = new ModalSandboxProvider(client); + const provider = new ModalSandboxProvider(client, "modal"); try { await provider.createSandbox(testConfig); @@ -458,7 +458,7 @@ describe("ModalSandboxProvider", () => { throw "string error"; // Throwing a string, not an Error }), }); - const provider = new ModalSandboxProvider(client); + const provider = new ModalSandboxProvider(client, "modal"); try { await provider.createSandbox(testConfig); @@ -478,7 +478,7 @@ describe("ModalSandboxProvider", () => { throw originalError; }), }); - const provider = new ModalSandboxProvider(client); + const provider = new ModalSandboxProvider(client, "modal"); try { await provider.createSandbox(testConfig); @@ -494,7 +494,7 @@ describe("ModalSandboxProvider", () => { throw new Error("timeout exceeded"); }), }); - const provider = new ModalSandboxProvider(client); + const provider = new ModalSandboxProvider(client, "modal"); try { await provider.createSandbox(testConfig); @@ -520,7 +520,7 @@ describe("ModalSandboxProvider", () => { const client = createMockModalClient({ createSandbox: vi.fn(async () => expectedResult), }); - const provider = new ModalSandboxProvider(client); + const provider = new ModalSandboxProvider(client, "modal"); const result = await provider.createSandbox({ ...testConfig, vncEnabled: true }); @@ -545,7 +545,7 @@ describe("ModalSandboxProvider", () => { }); await expect( - new ModalSandboxProvider(client).createSandbox({ + new ModalSandboxProvider(client, "modal").createSandbox({ ...testConfig, prebuiltImageId: "im-missing", }) @@ -565,7 +565,7 @@ describe("ModalSandboxProvider", () => { }), }); - const error = await new ModalSandboxProvider(client) + const error = await new ModalSandboxProvider(client, "modal") .createSandbox({ ...testConfig, prebuiltImageId: "im-valid" }) .catch((caught: unknown) => caught); @@ -577,7 +577,7 @@ describe("ModalSandboxProvider", () => { describe("image builds", () => { it("binds a created image-build sandbox before starting it", async () => { const client = createMockModalClient(); - const provider = new ModalSandboxProvider(client); + const provider = new ModalSandboxProvider(client, "modal"); const correlation = { request_id: "request-1", trace_id: "trace-1" }; const onProviderSessionCreated = vi.fn(async () => undefined); @@ -599,6 +599,8 @@ describe("ModalSandboxProvider", () => { expect(client.createImageBuildSandbox).toHaveBeenCalledWith( { + sandboxBackend: "modal", + resources: undefined, scopeKind: "repo", scopeId: "acme/repo", buildId: "build-123", @@ -638,7 +640,7 @@ describe("ModalSandboxProvider", () => { throw new ModalApiError("Modal API error: 502 Bad Gateway", 502); }), }); - const provider = new ModalSandboxProvider(client); + const provider = new ModalSandboxProvider(client, "modal"); try { await provider.restoreFromSnapshot({ @@ -666,7 +668,7 @@ describe("ModalSandboxProvider", () => { throw new ModalApiError("Modal API error: 401 Unauthorized", 401); }), }); - const provider = new ModalSandboxProvider(client); + const provider = new ModalSandboxProvider(client, "modal"); try { await provider.restoreFromSnapshot({ @@ -695,7 +697,7 @@ describe("ModalSandboxProvider", () => { throw modalError; }), }); - const provider = new ModalSandboxProvider(client); + const provider = new ModalSandboxProvider(client, "modal"); try { await provider.takeSnapshot({ @@ -718,7 +720,10 @@ describe("ModalSandboxProvider", () => { const snapshotBuildSandbox = vi.fn(async () => ({ imageId: "build-image-123", })); - const provider = new ModalSandboxProvider(createMockModalClient({ snapshotBuildSandbox })); + const provider = new ModalSandboxProvider( + createMockModalClient({ snapshotBuildSandbox }), + "modal" + ); await expect( provider.snapshotImageBuildSandbox({ @@ -742,7 +747,8 @@ describe("ModalSandboxProvider", () => { snapshotBuildSandbox: vi.fn(async () => { throw modalError; }), - }) + }), + "modal" ); await expect( @@ -764,7 +770,7 @@ describe("ModalSandboxProvider", () => { vncPassword: "vnc-pw", })), }); - const provider = new ModalSandboxProvider(client); + const provider = new ModalSandboxProvider(client, "modal"); const result = await provider.restoreFromSnapshot({ snapshotImageId: "img-123", diff --git a/packages/control-plane/src/sandbox/providers/modal-provider.ts b/packages/control-plane/src/sandbox/providers/modal-provider.ts index 8a38631e15..aad542dadd 100644 --- a/packages/control-plane/src/sandbox/providers/modal-provider.ts +++ b/packages/control-plane/src/sandbox/providers/modal-provider.ts @@ -6,13 +6,15 @@ */ import { ModalApiError } from "../client"; -import type { ModalClient } from "../client"; +import type { ModalClient, ModalBackend, CreateImageBuildSandboxResponse } from "../client"; +import type { SandboxSettings } from "@open-inspect/shared/types/integrations"; import type { CorrelationContext } from "../../logger"; import { supportsConfigurableSandboxTimeout } from "@open-inspect/shared/types/integrations"; import { DEFAULT_SANDBOX_TIMEOUT_SECONDS, PrebuiltImageUnavailableError, SandboxProviderError, + SandboxLaunchRejectedError, createVncAccess, signalUntilDeadline, type ImageBuildProviderTriggerConfig, @@ -37,6 +39,7 @@ interface StartModalImageBuildConfig { /** Modal extends the shared trigger contract with explicit SCM clone identity. */ export interface ModalImageBuildTriggerConfig extends ImageBuildProviderTriggerConfig { + resources?: Pick; cloneHost?: string; cloneUsername?: string; } @@ -76,7 +79,7 @@ export interface ModalImageBuildProvider { * @example * ```typescript * const client = createModalClient(secret, workspace, environmentWebSuffix); - * const provider = new ModalSandboxProvider(client); + * const provider = new ModalSandboxProvider(client, "modal"); * * try { * const result = await provider.createSandbox(config); @@ -88,17 +91,30 @@ export interface ModalImageBuildProvider { * ``` */ export class ModalSandboxProvider implements SandboxProvider, ModalImageBuildProvider { - readonly name = "modal"; + readonly name: ModalBackend; - readonly capabilities: SandboxProviderCapabilities = { - supportsSandboxTimeout: supportsConfigurableSandboxTimeout(this.name), - supportsSnapshots: true, - supportsRestore: true, - supportsPersistentResume: false, - supportsExplicitStop: true, - }; + readonly capabilities: SandboxProviderCapabilities; - constructor(private readonly client: ModalClient) {} + pendingSandboxReference(sessionId: string, sandboxId: string): string | undefined { + return this.name === "modal-vm" + ? `modal-vm-session:${JSON.stringify([sessionId, sandboxId])}` + : undefined; + } + + constructor( + private readonly client: ModalClient, + backend: ModalBackend + ) { + this.name = backend; + this.capabilities = { + supportsSandboxTimeout: supportsConfigurableSandboxTimeout(this.name), + supportsSnapshots: true, + snapshotRequiresShutdown: backend === "modal-vm", + supportsRestore: true, + supportsPersistentResume: false, + supportsExplicitStop: true, + }; + } /** * Create a new sandbox via Modal API. @@ -129,11 +145,14 @@ export class ModalSandboxProvider implements SandboxProvider, ModalImageBuildPro agentSlackNotifyEnabled: config.agentSlackNotifyEnabled, mcpServers: config.mcpServers, sandboxSettings: config.sandboxSettings, + sandboxBackend: this.name, + retireSandboxId: config.retireSandboxId, repositories: config.repositories, }, config.correlation ); + this.confirmSessionLaunch(result); return { sandboxId: result.sandboxId, providerObjectId: result.modalObjectId, @@ -185,11 +204,14 @@ export class ModalSandboxProvider implements SandboxProvider, ModalImageBuildPro agentSlackNotifyEnabled: config.agentSlackNotifyEnabled, mcpServers: config.mcpServers, sandboxSettings: config.sandboxSettings, + sandboxBackend: this.name, + retireSandboxId: config.retireSandboxId, repositories: config.repositories, }, config.correlation ); + this.confirmSessionLaunch(result); return { success: true, sandboxId: result.sandboxId, @@ -225,19 +247,46 @@ export class ModalSandboxProvider implements SandboxProvider, ModalImageBuildPro */ async takeSnapshot(config: SnapshotConfig): Promise { try { - const result = await this.client.snapshotSandbox( - { - providerObjectId: config.providerObjectId, - sessionId: config.sessionId, - signal: signalUntilDeadline(config.deadlineAtMs, config.signal), - deadlineAtMs: config.deadlineAtMs, - }, - config.correlation - ); + const request = { + providerObjectId: config.providerObjectId, + sessionId: config.sessionId, + sandboxBackend: this.name, + signal: signalUntilDeadline(config.deadlineAtMs, config.signal), + deadlineAtMs: config.deadlineAtMs, + }; + let result; + try { + result = await this.client.snapshotSandbox(request, config.correlation); + } catch (error) { + // The VM capture endpoint leaves the source alive until the control + // plane commits the image. Docker preparation is idempotent, so a + // lost response can safely retry the capture. + if ( + this.name !== "modal-vm" || + request.signal?.aborted || + (error instanceof ModalApiError && error.status < 500) + ) + throw error; + result = await this.client.snapshotSandbox(request, config.correlation); + } + if (this.name === "modal-vm") { + if (result.sourceStopped !== false) + throw new SandboxProviderError( + "Modal VM capture did not confirm source retention", + "permanent" + ); + if (!result.sourceObjectId) + throw new SandboxProviderError( + "Modal VM capture did not confirm its source ID", + "permanent" + ); + } return { success: true, imageId: result.imageId, + sourceStopped: result.sourceStopped === true, + sourceObjectId: result.sourceObjectId, }; } catch (error) { if (error instanceof ModalApiError) { @@ -298,10 +347,12 @@ export class ModalSandboxProvider implements SandboxProvider, ModalImageBuildPro private async createImageBuildSandbox( config: ModalImageBuildTriggerConfig - ): Promise<{ providerSessionId: string }> { + ): Promise { try { return await this.client.createImageBuildSandbox( { + sandboxBackend: this.name, + resources: config.resources, scopeKind: config.scopeKind, scopeId: config.scopeId, buildId: config.buildId, @@ -330,9 +381,34 @@ export class ModalSandboxProvider implements SandboxProvider, ModalImageBuildPro } } + private assertBackend(result: { sandboxBackend?: unknown }): void { + // Pre-backend Modal endpoints only created the standard sandbox and did not echo its backend. + const legacyStandard = this.name === "modal" && result.sandboxBackend === undefined; + if (result.sandboxBackend === this.name || legacyStandard) return; + throw new SandboxProviderError( + `Modal deployment did not confirm the ${this.name} backend; deploy compatible Modal endpoints`, + "permanent" + ); + } + + private confirmSessionLaunch(result: { modalObjectId?: string; sandboxBackend?: unknown }): void { + try { + this.assertBackend(result); + } catch (error) { + // The lifecycle must persist and fence this generation before any cleanup await. + throw new SandboxLaunchRejectedError( + error instanceof Error ? error.message : "Incompatible Modal allocation", + result.modalObjectId ?? null, + error instanceof Error ? error : undefined + ); + } + } + async triggerImageBuild(config: ModalImageBuildTriggerConfig): Promise { const created = await this.createImageBuildSandbox(config); + // Persist the handle for cleanup before checking compatibility. Binding does not start work. await config.onProviderSessionCreated(created.providerSessionId); + this.assertBackend(created); await this.startImageBuildSandbox({ buildId: config.buildId, providerSessionId: created.providerSessionId, @@ -392,6 +468,7 @@ export class ModalSandboxProvider implements SandboxProvider, ModalImageBuildPro * Classify an error as transient or permanent for circuit breaker handling. */ private classifyError(message: string, error: unknown): SandboxProviderError { + if (error instanceof SandboxProviderError) return error; if (SandboxProviderError.isTransientNetworkError(error)) { return new SandboxProviderError( `${message}: ${error instanceof Error ? error.message : String(error)}`, @@ -432,6 +509,9 @@ export class ModalSandboxProvider implements SandboxProvider, ModalImageBuildPro * @param client - ModalClient instance for API calls * @returns ModalSandboxProvider instance */ -export function createModalProvider(client: ModalClient): ModalSandboxProvider { - return new ModalSandboxProvider(client); +export function createModalProvider( + client: ModalClient, + backend: ModalBackend +): ModalSandboxProvider { + return new ModalSandboxProvider(client, backend); } diff --git a/packages/control-plane/src/sandbox/providers/vercel/provider.test.ts b/packages/control-plane/src/sandbox/providers/vercel/provider.test.ts index da56a1f353..a4ba615793 100644 --- a/packages/control-plane/src/sandbox/providers/vercel/provider.test.ts +++ b/packages/control-plane/src/sandbox/providers/vercel/provider.test.ts @@ -173,7 +173,7 @@ describe("VercelSandboxProvider", () => { supportsRestore: true, supportsPersistentResume: false, supportsExplicitStop: true, - snapshotStopsSandbox: true, + snapshotRequiresShutdown: true, }); }); diff --git a/packages/control-plane/src/sandbox/providers/vercel/provider.ts b/packages/control-plane/src/sandbox/providers/vercel/provider.ts index 90cb1e3c50..0bcd025852 100644 --- a/packages/control-plane/src/sandbox/providers/vercel/provider.ts +++ b/packages/control-plane/src/sandbox/providers/vercel/provider.ts @@ -92,7 +92,7 @@ export class VercelSandboxProvider implements SandboxProvider { supportsRestore: true, supportsPersistentResume: false, supportsExplicitStop: true, - snapshotStopsSandbox: true, + snapshotRequiresShutdown: true, }; constructor( diff --git a/packages/control-plane/src/session/client-command-facade.test.ts b/packages/control-plane/src/session/client-command-facade.test.ts index 8c65e43ec6..08791c3336 100644 --- a/packages/control-plane/src/session/client-command-facade.test.ts +++ b/packages/control-plane/src/session/client-command-facade.test.ts @@ -1,7 +1,8 @@ import { describe, expect, it, vi } from "vitest"; +import type { ShutdownRecoveryAction } from "@open-inspect/shared/types/sandbox-shutdown"; import { SessionClientCommandFacade } from "./client-command-facade"; -function createFacade(recover?: (action: "retry" | "restore_saved") => Promise) { +function createFacade(recover?: (action: ShutdownRecoveryAction) => Promise) { return new SessionClientCommandFacade( {} as never, {} as never, diff --git a/packages/control-plane/src/session/components.ts b/packages/control-plane/src/session/components.ts index bb36bdcb5e..1eb6a2ae9e 100644 --- a/packages/control-plane/src/session/components.ts +++ b/packages/control-plane/src/session/components.ts @@ -23,6 +23,7 @@ import { resolveAppName } from "@open-inspect/shared/app-name"; import { DEFAULT_MODEL } from "@open-inspect/shared/models"; +import { sandboxPromptBlockReason } from "@open-inspect/shared/types/sandbox-shutdown"; import { generateId, hashToken, encryptToken } from "../auth/crypto"; import { getUserAuth } from "../auth/user/runtime"; import { resolveSandboxBackendName } from "../sandbox/provider-name"; @@ -505,7 +506,8 @@ export function createSessionRuntime(platform: SessionPlatform, env: Env): Sessi alarmScheduler, executionStop, getExecutionTimeoutMs, - () => lifecycleManager.mayProcessQueuedWork() + () => lifecycleManager.mayProcessQueuedWork(), + () => sandboxPromptBlockReason(lifecycleManager.shutdownSnapshot()) ); // Tier 7 — services over the queue and lifecycle. @@ -981,6 +983,7 @@ export function createSessionRuntime(platform: SessionPlatform, env: Env): Sessi async () => { await wsManager.expireAuthorizationLeases(Date.now()); await alarmScheduler.rehydrate(); + await lifecycleManager.rearmRejectedStartupCleanupAlarm(); await terminalMessageProjection.rearm(); }, { @@ -1066,7 +1069,7 @@ function createLifecycleManager(deps: LifecycleManagerDeps): SandboxLifecycleMan }; const sandboxDashboardUrlBuilder = - sandboxBackend === "modal" + sandboxBackend === "modal" || sandboxBackend === "modal-vm" ? (providerObjectId: string) => resolveSandboxDashboardUrl(sandboxDashboardSettings, providerObjectId) : undefined; diff --git a/packages/control-plane/src/session/connection-authenticator.test.ts b/packages/control-plane/src/session/connection-authenticator.test.ts index ec4cb25169..784e407212 100644 --- a/packages/control-plane/src/session/connection-authenticator.test.ts +++ b/packages/control-plane/src/session/connection-authenticator.test.ts @@ -79,6 +79,7 @@ interface Harness { updateLastActivity: ReturnType; scheduleInactivityCheck: ReturnType; scheduleDisconnectCheck: ReturnType; + onRefusedReconnect: ReturnType; }; broadcast: ReturnType; submitted: string[]; @@ -116,6 +117,7 @@ function createHarness(opts: { updateLastActivity: vi.fn(), scheduleInactivityCheck: vi.fn(async () => undefined), scheduleDisconnectCheck: vi.fn(async () => undefined), + onRefusedReconnect: vi.fn((): "retry" | "exit" => "exit"), }; const broadcast = vi.fn(); const submitted: string[] = []; @@ -214,14 +216,35 @@ describe("SessionConnectionAuthenticator.authorize", () => { it("rejects a terminal session with 410 on a read taken after authentication", async () => { const h = createHarness({ sandbox: await sandboxRow(), session: sessionRow("cancelled") }); + h.lifecycleManager.onRefusedReconnect.mockReturnValue("retry"); const decision = await h.authenticator.authorize( upgradeRequest({ sandbox: true, token: TOKEN, sandboxId: SANDBOX_ID }) ); + // A cancelled session's sandbox is destroyed, never kept for a save. expect(await rejection(decision)).toEqual({ status: 410, body: "Session is terminal" }); + expect(h.lifecycleManager.onRefusedReconnect).not.toHaveBeenCalled(); }); + it.each([ + { instruction: "retry" as const, expected: { status: 503, body: "Sandbox is being saved" } }, + { instruction: "exit" as const, expected: { status: 410, body: "Session is terminal" } }, + ])( + "answers an archived session's sandbox with $expected.status when told to $instruction", + async ({ instruction, expected }) => { + const h = createHarness({ sandbox: await sandboxRow(), session: sessionRow("archived") }); + h.lifecycleManager.onRefusedReconnect.mockReturnValue(instruction); + + const decision = await h.authenticator.authorize( + upgradeRequest({ sandbox: true, token: TOKEN, sandboxId: SANDBOX_ID }) + ); + + expect(await rejection(decision)).toEqual(expected); + expect(h.lifecycleManager.onRefusedReconnect).toHaveBeenCalledOnce(); + } + ); + it("rejects a sandbox that stopped during the token hash with 410", async () => { const row = await sandboxRow(); const h = createHarness({ @@ -236,6 +259,34 @@ describe("SessionConnectionAuthenticator.authorize", () => { expect(await rejection(decision)).toEqual({ status: 410, body: "Sandbox is stopped" }); }); + it("tells a stale sandbox that a save still needs to retry instead of exiting", async () => { + const h = createHarness({ sandbox: await sandboxRow({ status: "stale" }) }); + h.lifecycleManager.onRefusedReconnect.mockReturnValue("retry"); + + const decision = await h.authenticator.authorize( + upgradeRequest({ sandbox: true, token: TOKEN, sandboxId: SANDBOX_ID }) + ); + + expect(await rejection(decision)).toEqual({ status: 503, body: "Sandbox is being saved" }); + expect(h.lifecycleManager.onRefusedReconnect).toHaveBeenCalledOnce(); + }); + + it("does not keep a superseded generation up for the current generation's save", async () => { + const row = await sandboxRow({ status: "stale" }); + const h = createHarness({ + sandbox: row, + duringTokenHash: () => ({ ...row, created_at: row.created_at + 1 }), + }); + h.lifecycleManager.onRefusedReconnect.mockReturnValue("retry"); + + const decision = await h.authenticator.authorize( + upgradeRequest({ sandbox: true, token: TOKEN, sandboxId: SANDBOX_ID }) + ); + + expect(await rejection(decision)).toEqual({ status: 410, body: "Sandbox is stopped" }); + expect(h.lifecycleManager.onRefusedReconnect).not.toHaveBeenCalled(); + }); + it("rejects credentials rotated during the token hash with 403", async () => { const row = await sandboxRow(); const h = createHarness({ diff --git a/packages/control-plane/src/session/connection-authenticator.ts b/packages/control-plane/src/session/connection-authenticator.ts index 3276bd6b19..a93c45e342 100644 --- a/packages/control-plane/src/session/connection-authenticator.ts +++ b/packages/control-plane/src/session/connection-authenticator.ts @@ -155,6 +155,18 @@ export class SessionConnectionAuthenticator implements SessionUpgradeAdmission { return reject("Unauthorized: Invalid auth token", 401); } + // A refused bridge exits and its sandbox shuts itself down. While a save + // still needs this sandbox, the bridge is told to retry instead; the save + // stops the sandbox once it is done with it. + const refusedReconnect = (): "retry" | "exit" => { + const current = sandboxRepository.getSandbox(); + return current !== null && + current.modal_sandbox_id === expectedSandboxId && + current.created_at === sandbox.created_at + ? this.deps.lifecycleManager.onRefusedReconnect() + : "exit"; + }; + // Reject connection if the session itself is closed for good. Narrower // than "not active": `completed` and `failed` sessions are idle, not // over — warm-on-typing spawns a sandbox for one before the follow-up @@ -165,30 +177,37 @@ export class SessionConnectionAuthenticator implements SessionUpgradeAdmission { // request is suspended. Admission needs a fresh, synchronous read. const currentSession = sessionCoreRepository.getSession(); if (currentSession && !isSessionPromptable(currentSession.status)) { + // An archived session's sandbox is being saved; a cancelled one is destroyed. + const instruction = currentSession.status === "archived" ? refusedReconnect() : "exit"; log.warn("ws.connect", { event: "ws.connect", ws_type: "sandbox", outcome: "rejected", - reject_reason: "session_terminal", + reject_reason: instruction === "retry" ? "sandbox_preserving" : "session_terminal", session_status: currentSession.status, duration_ms: Date.now() - wsStartTime, }); - return reject("Session is terminal", 410); + return instruction === "retry" + ? reject("Sandbox is being saved", 503) + : reject("Session is terminal", 410); } const currentSandbox = sandboxRepository.getSandbox(); // Deliberately narrower than isDeadSandboxStatus: a "failed" sandbox may // still connect after a slow boot and self-heal by becoming ready. if (currentSandbox && isSandboxReconnectBlockedStatus(currentSandbox.status)) { + const instruction = refusedReconnect(); log.warn("ws.connect", { event: "ws.connect", ws_type: "sandbox", outcome: "rejected", - reject_reason: "sandbox_stopped", + reject_reason: instruction === "retry" ? "sandbox_preserving" : "sandbox_stopped", sandbox_status: currentSandbox.status, duration_ms: Date.now() - wsStartTime, }); - return reject("Sandbox is stopped", 410); + return instruction === "retry" + ? reject("Sandbox is being saved", 503) + : reject("Sandbox is stopped", 410); } if ( !currentSandbox || diff --git a/packages/control-plane/src/session/http/handlers/child-sessions.handler.test.ts b/packages/control-plane/src/session/http/handlers/child-sessions.handler.test.ts index 023b6dfbfb..8e4eb32e19 100644 --- a/packages/control-plane/src/session/http/handlers/child-sessions.handler.test.ts +++ b/packages/control-plane/src/session/http/handlers/child-sessions.handler.test.ts @@ -1,7 +1,11 @@ import { describe, expect, it, vi } from "vitest"; import { MAX_CHILD_FOLLOW_UP_PROMPT_CHARS } from "@open-inspect/shared/types/session-api"; import { ChildSessionsHandler } from "./child-sessions.handler"; -import { PromptQueueFullError, SessionNotPromptableError } from "../../message-queue"; +import { + PromptQueueFullError, + SandboxPromptBlockedError, + SessionNotPromptableError, +} from "../../message-queue"; import type { ParticipantRow, SessionRow } from "../../types"; import type { ParticipantRepository } from "../../participant-repository"; import type { MessageRepository } from "../../message-repository"; @@ -234,6 +238,23 @@ describe("ChildSessionsHandler", () => { error: "Cannot prompt a archived session", }); }); + + it("maps a sandbox safety hold to 409", async () => { + const { handler, getSession, repository, enqueuePrompt } = createHandler(); + getSession.mockReturnValue(createSession({ parent_session_id: "parent-1" })); + repository.listParticipants.mockReturnValue([createParticipant()]); + enqueuePrompt.mockRejectedValue(new SandboxPromptBlockedError("Start a new session.")); + + const response = await handler.parentPrompt( + request({ parentSessionId: "parent-1", content: "Continue" }) + ); + + expect(response.status).toBe(409); + await expect(response.json()).resolves.toEqual({ + error: "Start a new session.", + code: "SANDBOX_RECOVERY_REQUIRED", + }); + }); }); it("returns 404 when session is missing for spawn context", async () => { diff --git a/packages/control-plane/src/session/http/handlers/child-sessions.handler.ts b/packages/control-plane/src/session/http/handlers/child-sessions.handler.ts index 211696d90d..d7ceae50bb 100644 --- a/packages/control-plane/src/session/http/handlers/child-sessions.handler.ts +++ b/packages/control-plane/src/session/http/handlers/child-sessions.handler.ts @@ -5,7 +5,11 @@ import { z } from "zod"; import { sessionStatusSchema } from "@open-inspect/shared/types/sessions"; import { parsePersistedSandboxSettings } from "../../../sandbox/settings"; import type { SessionMessenger } from "../../messenger"; -import { PromptQueueFullError, SessionNotPromptableError } from "../../message-queue"; +import { + PromptQueueFullError, + SandboxPromptBlockedError, + SessionNotPromptableError, +} from "../../message-queue"; import type { MessageRepository } from "../../message-repository"; import type { ParticipantRepository } from "../../participant-repository"; import type { SessionCoreRepository } from "../../session-core-repository"; @@ -165,6 +169,12 @@ export class ChildSessionsHandler { if (error instanceof SessionNotPromptableError) { return Response.json({ error: error.message }, { status: 409 }); } + if (error instanceof SandboxPromptBlockedError) { + return Response.json( + { error: error.message, code: "SANDBOX_RECOVERY_REQUIRED" }, + { status: 409 } + ); + } if (error instanceof PromptQueueFullError) { return Response.json({ error: "Child prompt queue is full" }, { status: 429 }); } diff --git a/packages/control-plane/src/session/http/handlers/child-summary.handler.test.ts b/packages/control-plane/src/session/http/handlers/child-summary.handler.test.ts index b72a34422e..d6240b3f4b 100644 --- a/packages/control-plane/src/session/http/handlers/child-summary.handler.test.ts +++ b/packages/control-plane/src/session/http/handlers/child-summary.handler.test.ts @@ -75,6 +75,7 @@ function createSandbox(overrides: Partial = {}): SandboxRow { boot_phase: null, boot_seq: null, fenced: 0, + startup_rejected: 0, created_at: 1, ...overrides, }; diff --git a/packages/control-plane/src/session/http/handlers/messages.handler.test.ts b/packages/control-plane/src/session/http/handlers/messages.handler.test.ts index ec8b1164c8..8feb4784bd 100644 --- a/packages/control-plane/src/session/http/handlers/messages.handler.test.ts +++ b/packages/control-plane/src/session/http/handlers/messages.handler.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it, vi } from "vitest"; import type { Logger } from "../../../logger"; import { MessagesHandler } from "./messages.handler"; +import { SandboxPromptBlockedError } from "../../message-queue"; import type { MessageService } from "../../services/message.service"; import { MAX_WEB_PROMPT_CHARS } from "@open-inspect/shared/types/prompts"; @@ -31,6 +32,26 @@ function createHandler() { } describe("MessagesHandler", () => { + it("returns a recoverable 409 when sandbox safety blocks prompt admission", async () => { + const { handler, messageService, log } = createHandler(); + vi.mocked(messageService.enqueuePrompt).mockRejectedValue( + new SandboxPromptBlockedError("Start a new session to continue.") + ); + + const response = await handler.enqueuePrompt( + new Request("http://internal/internal/prompt", { + method: "POST", + body: JSON.stringify({ content: "Continue", authorId: "user-1", source: "web" }), + }), + log + ); + + expect(response.status).toBe(409); + await expect(response.json()).resolves.toEqual({ + code: "SANDBOX_RECOVERY_REQUIRED", + error: "Start a new session to continue.", + }); + }); it("enqueues prompt and returns queued response", async () => { const { handler, messageService, log } = createHandler(); vi.mocked(messageService.enqueuePrompt).mockResolvedValue({ diff --git a/packages/control-plane/src/session/http/handlers/messages.handler.ts b/packages/control-plane/src/session/http/handlers/messages.handler.ts index 0ebd026177..7a47b02566 100644 --- a/packages/control-plane/src/session/http/handlers/messages.handler.ts +++ b/packages/control-plane/src/session/http/handlers/messages.handler.ts @@ -17,6 +17,7 @@ import { HarnessModelIncompatibleError, PromptRequestConflictError, SessionNotPromptableError, + SandboxPromptBlockedError, } from "../../message-queue"; /** @@ -44,6 +45,12 @@ export class MessagesHandler { if (error instanceof SessionNotPromptableError) { return Response.json({ error: error.message }, { status: 409 }); } + if (error instanceof SandboxPromptBlockedError) { + return Response.json( + { error: error.message, code: "SANDBOX_RECOVERY_REQUIRED" }, + { status: 409 } + ); + } if (error instanceof BudgetExhaustedError) { return Response.json({ error: error.message, code: "BUDGET_EXHAUSTED" }, { status: 409 }); } diff --git a/packages/control-plane/src/session/http/handlers/session-lifecycle.handler.test.ts b/packages/control-plane/src/session/http/handlers/session-lifecycle.handler.test.ts index 903916b6f7..026df7ad7e 100644 --- a/packages/control-plane/src/session/http/handlers/session-lifecycle.handler.test.ts +++ b/packages/control-plane/src/session/http/handlers/session-lifecycle.handler.test.ts @@ -69,6 +69,7 @@ function createSandbox(overrides: Partial = {}): SandboxRow { boot_phase: null, boot_seq: null, fenced: 0, + startup_rejected: 0, created_at: 1, ...overrides, }; @@ -98,6 +99,7 @@ function createHandler() { const applySessionTitleUpdate = vi.fn((title: string) => ({ ok: true as const, title })); const cancelSession = vi.fn(); const cancelSandbox = vi.fn(); + const preserveForArchive = vi.fn(async () => undefined); const lifecycleHandler = new SessionLifecycleHandler( repository as unknown as SessionCoreRepository, @@ -105,7 +107,7 @@ function createHandler() { repository as unknown as MessageRepository, statusService, { applySessionTitleUpdate } as unknown as SessionTitleService, - { cancelSandbox }, + { cancelSandbox, preserveForArchive }, "session-do-id", cancelSession ); @@ -132,6 +134,7 @@ function createHandler() { applySessionTitleUpdate, cancelSession, cancelSandbox, + preserveForArchive, }; } @@ -277,7 +280,7 @@ describe("SessionLifecycleHandler", () => { }); it("archives successfully without participant authorization", async () => { - const { handler, getSession, transition } = createHandler(); + const { handler, getSession, transition, preserveForArchive } = createHandler(); getSession.mockReturnValue(createSession()); transition.mockResolvedValue(true); @@ -292,6 +295,11 @@ describe("SessionLifecycleHandler", () => { expect(response.status).toBe(200); expect(await response.json()).toEqual({ status: "archived", outcome: "archived" }); expect(transition).toHaveBeenCalledWith("archived"); + // An archived session's reconnects are refused, so its sandbox is saved now. + expect(preserveForArchive).toHaveBeenCalledOnce(); + expect(transition.mock.invocationCallOrder[0]).toBeLessThan( + preserveForArchive.mock.invocationCallOrder[0] + ); }); it("archives a draft that was never prompted", async () => { @@ -397,7 +405,7 @@ describe("SessionLifecycleHandler", () => { }); it("returns 409 when archiving a session with queued work", async () => { - const { handler, getSession, repository, transition } = createHandler(); + const { handler, getSession, repository, transition, preserveForArchive } = createHandler(); getSession.mockReturnValue(createSession()); repository.getPendingOrProcessingCount.mockReturnValue(1); @@ -410,6 +418,7 @@ describe("SessionLifecycleHandler", () => { expect(response.status).toBe(409); expect(transition).not.toHaveBeenCalled(); + expect(preserveForArchive).not.toHaveBeenCalled(); }); it("returns 409 when archiving a cancelled session", async () => { diff --git a/packages/control-plane/src/session/http/handlers/session-lifecycle.handler.ts b/packages/control-plane/src/session/http/handlers/session-lifecycle.handler.ts index e4ac484f93..2f0c659140 100644 --- a/packages/control-plane/src/session/http/handlers/session-lifecycle.handler.ts +++ b/packages/control-plane/src/session/http/handlers/session-lifecycle.handler.ts @@ -163,6 +163,7 @@ export class SessionLifecycleHandler { } await this.statusService.transition("archived"); + await this.sandboxLifecycle.preserveForArchive(); try { await this.statusService.confirmIndexStatus("archived"); } catch { @@ -259,7 +260,7 @@ export class SessionLifecycleHandler { await this.cancelSession(); - this.sandboxLifecycle.cancelSandbox(); + await this.sandboxLifecycle.cancelSandbox(); return Response.json({ status: "cancelled" }); } diff --git a/packages/control-plane/src/session/message-queue.test.ts b/packages/control-plane/src/session/message-queue.test.ts index b06629fa3c..732cac10fd 100644 --- a/packages/control-plane/src/session/message-queue.test.ts +++ b/packages/control-plane/src/session/message-queue.test.ts @@ -136,7 +136,10 @@ it("creates a canonical SHA-256 web prompt fingerprint", async () => { ).resolves.toBe(fingerprint); }); -function buildQueue(mayDispatch: () => boolean = () => true) { +function buildQueue( + mayDispatch: () => boolean = () => true, + getSandboxPromptBlockReason: () => string | null = () => null +) { // Mutable so tests can pin that the deadline honors the value current at // dispatch time — the thunk exists because settings can be persisted after // the queue is constructed. @@ -299,7 +302,8 @@ function buildQueue(mayDispatch: () => boolean = () => true) { alarmScheduler, executionStop, () => executionTimeoutMs, - mayDispatch + mayDispatch, + getSandboxPromptBlockReason ); return { @@ -328,6 +332,56 @@ function buildQueue(mayDispatch: () => boolean = () => true) { } describe("SessionMessageQueue", () => { + it("rejects new websocket and API prompts during a failed safety hold", async () => { + const h = buildQueue( + () => false, + () => "Start a new session to continue." + ); + const ws = {} as WebSocket; + h.participantService.getByUserId.mockReturnValue(null as unknown as ParticipantRow); + + await h.queue.handlePromptMessage(ws, createClientInfo(), { + content: "Continue", + clientRequestId: "request-1", + }); + await expect( + h.queue.enqueuePromptFromApi({ content: "Continue", authorId: "user-1", source: "agent" }) + ).rejects.toMatchObject({ name: "SandboxPromptBlockedError" }); + + expect(h.wsManager.send).toHaveBeenCalledWith( + ws, + expect.objectContaining({ + type: "error", + code: "SANDBOX_RECOVERY_REQUIRED", + clientRequestId: "request-1", + message: "Start a new session to continue.", + }) + ); + expect(h.participantService.create).not.toHaveBeenCalled(); + expect(h.repository.createMessageWithAttachments).not.toHaveBeenCalled(); + expect(h.sessionStatus.transition).not.toHaveBeenCalled(); + }); + + it("rechecks the safety hold after asynchronous prompt fingerprinting", async () => { + let held = false; + const h = buildQueue( + () => true, + () => (held ? "Sandbox recovery required" : null) + ); + const ws = {} as WebSocket; + const handling = h.queue.handlePromptMessage(ws, createClientInfo(), { + content: "Continue", + clientRequestId: "request-1", + }); + held = true; + await handling; + + expect(h.repository.createMessageWithAttachments).not.toHaveBeenCalled(); + expect(h.wsManager.send).toHaveBeenCalledWith( + ws, + expect.objectContaining({ code: "SANDBOX_RECOVERY_REQUIRED" }) + ); + }); it("cannot dispatch while final-cost settlement waits for terminal projection", async () => { const h = buildQueue(); const session = createSession({ total_cost: 9, max_cost_usd: 10 }); @@ -445,6 +499,7 @@ describe("SessionMessageQueue", () => { attemptLimit: 10, windowStart: expect.any(Number), sessionClosed: false, + sandboxRecoveryRequired: false, }); expect(h.repository.createEvent).not.toHaveBeenCalled(); expect(h.sessionStatus.transition).toHaveBeenCalledWith("active"); @@ -510,6 +565,38 @@ describe("SessionMessageQueue", () => { expect(h.repository.updateParticipantCoalesce).not.toHaveBeenCalled(); }); + it("rejects new Autofix feedback during a failed safety hold", async () => { + const h = buildQueue( + () => false, + () => "Sandbox recovery required" + ); + h.repository.admitAutofixMessage.mockReturnValue({ + kind: "rejected", + reason: "sandbox_recovery_required", + }); + + const result = await h.queue.enqueueAutofix({ + type: "enqueue_feedback", + feedbackKey: "github:review:held", + pullRequest: { repositoryId: "99", number: 42, artifactId: "artifact-1" }, + prompt: "Address the feedback", + author: { id: "7", login: "alice" }, + origin: { + kind: "review", + authorType: "human", + feedbackUrl: "https://github.com/acme/widgets/pull/42#pullrequestreview-held", + }, + attemptLimit: 10, + }); + + expect(result).toEqual({ kind: "rejected", reason: "sandbox_recovery_required" }); + expect(h.repository.admitAutofixMessage).toHaveBeenCalledWith( + expect.objectContaining({ sandboxRecoveryRequired: true }) + ); + expect(h.participantService.create).not.toHaveBeenCalled(); + expect(h.sessionStatus.transition).not.toHaveBeenCalled(); + }); + it("returns a duplicate without re-driving it in a closed session", async () => { const h = buildQueue(); h.repository.getSession.mockReturnValue(createSession({ status: "archived" })); diff --git a/packages/control-plane/src/session/message-queue.ts b/packages/control-plane/src/session/message-queue.ts index fdb88c01f2..54ca189d09 100644 --- a/packages/control-plane/src/session/message-queue.ts +++ b/packages/control-plane/src/session/message-queue.ts @@ -77,6 +77,13 @@ export class SessionNotPromptableError extends Error { } } +export class SandboxPromptBlockedError extends Error { + constructor(message: string) { + super(message); + this.name = "SandboxPromptBlockedError"; + } +} + export class BudgetExhaustedError extends Error { constructor() { super( @@ -164,13 +171,15 @@ export class SessionMessageQueue { private readonly executionStop: ExecutionStopCoordinator, /** Resolved per use so it honors settings persisted after construction. */ private readonly getExecutionTimeoutMs: () => number, - private readonly mayDispatch: () => boolean = () => true + private readonly mayDispatch: () => boolean, + private readonly getSandboxPromptBlockReason: () => string | null ) {} async enqueueAutofix( command: Extract ): Promise { const session = this.repository.getSession(); + const sandboxRecoveryRequired = this.getSandboxPromptBlockReason() !== null; const userId = `github:${command.author.id}`; const now = Date.now(); const admission = this.messageRepository.admitAutofixMessage({ @@ -199,6 +208,7 @@ export class SessionMessageQueue { attemptLimit: command.attemptLimit, windowStart: now - AUTOFIX_ATTEMPT_WINDOW_MS, sessionClosed: !session || session.status === "archived" || session.status === "cancelled", + sandboxRecoveryRequired, }); if (admission.kind === "rejected") return admission; @@ -229,6 +239,7 @@ export class SessionMessageQueue { const session = this.repository.getSession(); if (!session || session.status === "archived" || session.status === "cancelled") return; + if (this.getSandboxPromptBlockReason()) return; await this.sessionStatus.transition("active"); await this.processMessageQueue(); @@ -242,6 +253,7 @@ export class SessionMessageQueue { let enqueued: EnqueuedPrompt; try { this.assertPromptableSession(); + this.assertSandboxAcceptingPrompts(); let participant = this.participantRepository.getParticipantById(client.participantId); participant ??= this.participantService.getByUserId(client.userId); if (!participant) { @@ -278,6 +290,15 @@ export class SessionMessageQueue { }); return; } + if (error instanceof SandboxPromptBlockedError) { + this.wsManager.send(ws, { + type: "error", + code: "SANDBOX_RECOVERY_REQUIRED", + message: error.message, + clientRequestId: data.clientRequestId, + }); + return; + } if (error instanceof PromptQueueFullError) { this.wsManager.send(ws, { type: "error", @@ -679,6 +700,7 @@ export class SessionMessageQueue { data: EnqueuePromptRequest ): Promise<{ messageId: string; status: "queued" }> { this.assertPromptableSession(); + this.assertSandboxAcceptingPrompts(); this.assertBudgetAvailable(); this.assertQueueCapacity(); let participant = this.participantService.getByUserId(data.authorId); @@ -738,6 +760,7 @@ export class SessionMessageQueue { // cancel or archive can land while this request is suspended, so the // session is read after it, not before. this.assertPromptableSession(); + this.assertSandboxAcceptingPrompts(); const queueDepthBefore = this.messageRepository.getPendingOrProcessingCount(); if (data.clientRequestId) { const existing = this.messageRepository.getMessageByClientRequestId(data.clientRequestId); @@ -878,6 +901,11 @@ export class SessionMessageQueue { } } + private assertSandboxAcceptingPrompts(): void { + const reason = this.getSandboxPromptBlockReason(); + if (reason) throw new SandboxPromptBlockedError(reason); + } + private assertQueueCapacity( queueDepth = this.messageRepository.getPendingOrProcessingCount() ): void { diff --git a/packages/control-plane/src/session/message-repository.test.ts b/packages/control-plane/src/session/message-repository.test.ts index 545739f2e4..5ffd590532 100644 --- a/packages/control-plane/src/session/message-repository.test.ts +++ b/packages/control-plane/src/session/message-repository.test.ts @@ -286,6 +286,29 @@ describe("MessageRepository", () => { expect(mock.calls).toHaveLength(2); }); + it("rejects new Autofix feedback during a sandbox safety hold", () => { + expect( + repository.admitAutofixMessage({ + message: { + id: "msg-new", + authorId: "p-1", + content: "Fix feedback", + source: "github", + status: "pending", + createdAt: 2000, + }, + feedbackKey: "github:review:held", + pullRequestKey: "github:99:42", + originContext: "{}", + attemptLimit: 3, + windowStart: 1000, + sessionClosed: false, + sandboxRecoveryRequired: true, + }) + ).toEqual({ kind: "rejected", reason: "sandbox_recovery_required" }); + expect(mock.calls).toHaveLength(2); + }); + it("rejects new Autofix feedback when the session budget is exhausted", () => { mock.setData(`SELECT budget_exhausted FROM session LIMIT 1`, [{ budget_exhausted: 1 }]); diff --git a/packages/control-plane/src/session/message-repository.ts b/packages/control-plane/src/session/message-repository.ts index 99a77d12b5..47fa38975e 100644 --- a/packages/control-plane/src/session/message-repository.ts +++ b/packages/control-plane/src/session/message-repository.ts @@ -68,6 +68,7 @@ export interface AdmitAutofixMessageData { attemptLimit: number | null; windowStart: number; sessionClosed: boolean; + sandboxRecoveryRequired?: boolean; } export type AutofixMessageAdmission = @@ -75,7 +76,12 @@ export type AutofixMessageAdmission = | { kind: "duplicate"; messageId: string } | { kind: "rejected"; - reason: "session_closed" | "budget_exhausted" | "queue_full" | "attempt_limit"; + reason: + | "session_closed" + | "sandbox_recovery_required" + | "budget_exhausted" + | "queue_full" + | "attempt_limit"; }; /** Options for listing messages. */ @@ -237,6 +243,9 @@ export class MessageRepository { if (data.sessionClosed) { return { kind: "rejected", reason: "session_closed" }; } + if (data.sandboxRecoveryRequired) { + return { kind: "rejected", reason: "sandbox_recovery_required" }; + } if (this.getPendingOrProcessingCount() >= MAX_UNFINISHED_PROMPTS) { return { kind: "rejected", reason: "queue_full" }; } diff --git a/packages/control-plane/src/session/sandbox-access.ts b/packages/control-plane/src/session/sandbox-access.ts index 387c5798bd..f8ea09c28d 100644 --- a/packages/control-plane/src/session/sandbox-access.ts +++ b/packages/control-plane/src/session/sandbox-access.ts @@ -80,7 +80,8 @@ export function resolveSandboxDashboardUrl( settings: SandboxDashboardSettings, providerObjectId: string | null | undefined ): string | null { - if (resolveSandboxBackendName(settings.sandboxProvider) !== "modal") return null; + const backend = resolveSandboxBackendName(settings.sandboxProvider); + if (backend !== "modal" && backend !== "modal-vm") return null; return buildModalSandboxDashboardUrl({ workspace: settings.modalWorkspace, modalEnvironment: settings.modalEnvironment, diff --git a/packages/control-plane/src/session/sandbox-ports.ts b/packages/control-plane/src/session/sandbox-ports.ts index 29cc15d074..856f888f7a 100644 --- a/packages/control-plane/src/session/sandbox-ports.ts +++ b/packages/control-plane/src/session/sandbox-ports.ts @@ -34,6 +34,11 @@ export interface SandboxShutdownStorage extends SandboxStateReader { from: SandboxStatus, to: SandboxStatus ): boolean; + /** + * Stop the named generation and forget its snapshot and provider handle, so + * the next start neither restores nor resumes it. Reports whether it applied. + */ + discardSandboxState(generation: { sandboxId: string | null; createdAt: number }): boolean; } /** Aggregate initialization is separate from transitions of an existing sandbox. */ diff --git a/packages/control-plane/src/session/sandbox-repository.test.ts b/packages/control-plane/src/session/sandbox-repository.test.ts index 3ad1571698..0deb0a907f 100644 --- a/packages/control-plane/src/session/sandbox-repository.test.ts +++ b/packages/control-plane/src/session/sandbox-repository.test.ts @@ -36,6 +36,7 @@ function sandboxRow(overrides: Partial = {}): SandboxRow { boot_phase: null, boot_seq: null, fenced: 0, + startup_rejected: 0, created_at: 1000, ...overrides, }; @@ -469,6 +470,46 @@ describe("SandboxRepository boot state (SQLite)", () => { return { db, sql, repository, set }; } + describe("rejectProviderStartup", () => { + const generation = { sandboxId: "sb-1", createdAt: 1000 }; + + it.each(["spawning", "connecting", "ready", "failed", "stopped", "stale"])( + "fences %s and persists cleanup responsibility", + (status) => { + const { repository, set } = createSqliteRepository(); + set( + "status = ?, modal_sandbox_id = 'sb-1', fenced = ?, auth_token_hash = 'hash', active_socket_id = 'socket'", + status, + status === "failed" ? 1 : 0 + ); + expect(repository.rejectProviderStartup(generation, "sb-rejected")).toBe( + ["spawning", "connecting", "ready"].includes(status) ? "failed" : "retained" + ); + expect(repository.getSandbox()).toMatchObject({ + status: ["stopped", "stale"].includes(status) ? status : "failed", + startup_rejected: 1, + fenced: 1, + modal_object_id: "sb-rejected", + auth_token_hash: "", + auth_token: null, + active_socket_id: "", + }); + expect(repository.markSandboxReady(generation)).toBe(false); + } + ); + + it("records confirmed cleanup without retaining an obligation and rejects superseded writes", () => { + const { repository, set } = createSqliteRepository(); + set("status = 'connecting', modal_sandbox_id = 'sb-1', modal_object_id = 'old'"); + expect(repository.rejectProviderStartup(generation, null)).toBe("failed"); + expect(repository.getSandbox()?.modal_object_id).toBeNull(); + expect(repository.rejectProviderStartup({ ...generation, createdAt: 999 }, "late")).toBe( + "superseded" + ); + expect(repository.getSandbox()?.modal_object_id).toBeNull(); + }); + }); + describe("commitProviderStartup", () => { const generation = { sandboxId: "sb-1", createdAt: 1000 }; diff --git a/packages/control-plane/src/session/sandbox-repository.ts b/packages/control-plane/src/session/sandbox-repository.ts index 86b1e3e2b5..f08f73e8eb 100644 --- a/packages/control-plane/src/session/sandbox-repository.ts +++ b/packages/control-plane/src/session/sandbox-repository.ts @@ -170,6 +170,31 @@ export class SandboxRepository { return (result.rowsWritten ?? 0) > 0; } + /** Persist cleanup responsibility while permanently revoking a rejected generation. */ + rejectProviderStartup( + generation: { sandboxId: string | null; createdAt: number }, + providerObjectId: string | null + ): "failed" | "retained" | "superseded" { + const assignments = `modal_object_id = ?, fenced = 1, startup_rejected = 1, + auth_token_hash = '', auth_token = NULL, active_socket_id = ''`; + const identity = `id = (SELECT id FROM sandbox LIMIT 1) + AND modal_sandbox_id IS ? AND created_at = ?`; + const args = [providerObjectId, generation.sandboxId, generation.createdAt]; + const failed = this.sql + .exec( + `UPDATE sandbox SET ${assignments}, status = 'failed' + WHERE ${identity} AND status IN ('spawning', 'connecting', 'ready') + RETURNING id`, + ...args + ) + .toArray(); + if (failed.length) return "failed"; + const retained = this.sql + .exec(`UPDATE sandbox SET ${assignments} WHERE ${identity} RETURNING id`, ...args) + .toArray(); + return retained.length ? "retained" : "superseded"; + } + commitProviderStartup( generation: { sandboxId: string | null; createdAt: number }, providerObjectId: string | null, @@ -287,7 +312,7 @@ export class SandboxRepository { active_socket_id = '', boot_phase = NULL, boot_seq = NULL, - fenced = 0 + fenced = 0, startup_rejected = 0 WHERE id = (SELECT id FROM sandbox LIMIT 1)`, data.status, data.createdAt, @@ -342,7 +367,7 @@ export class SandboxRepository { last_heartbeat = NULL, boot_phase = NULL, boot_seq = NULL, - fenced = 0 + fenced = 0, startup_rejected = 0 WHERE id = (SELECT id FROM sandbox LIMIT 1)`, data.status, data.createdAt @@ -420,6 +445,24 @@ export class SandboxRepository { return (result.rowsWritten ?? 0) > 0; } + /** + * Stop the generation and forget its snapshot and provider handle, so the + * next start is a fresh spawn: no restore from the snapshot and no resume of + * the provider object. Applies only while the row is still that generation. + */ + discardSandboxState(generation: { sandboxId: string | null; createdAt: number }): boolean { + const result = this.sql.exec( + `UPDATE sandbox SET status = 'stopped', snapshot_image_id = NULL, + snapshot_runtime_version = NULL, modal_object_id = NULL + WHERE id = (SELECT id FROM sandbox LIMIT 1) + AND modal_sandbox_id IS ? AND created_at = ?`, + generation.sandboxId, + generation.createdAt + ); + result.toArray(); + return (result.rowsWritten ?? 0) > 0; + } + /** * Set the runtime version describing the sandbox's current filesystem. * diff --git a/packages/control-plane/src/session/sandbox-shutdown-repository.ts b/packages/control-plane/src/session/sandbox-shutdown-repository.ts index f708542e27..f090099ddb 100644 --- a/packages/control-plane/src/session/sandbox-shutdown-repository.ts +++ b/packages/control-plane/src/session/sandbox-shutdown-repository.ts @@ -7,6 +7,7 @@ import { SessionStorageIntegrityError } from "./types"; const receiptSchema = z.object({ kind: z.enum(["snapshot", "retained"]), artifactId: z.string().min(1), + sourceObjectId: z.string().min(1).optional(), provider: z.string(), savedAtMs: z.number(), runtimeVersion: z.string().nullable(), @@ -27,6 +28,8 @@ const stateSchema = sandboxShutdownSchema lifecyclePolicy: z.enum(["confirmed", "legacy"]).optional(), restoreInvoked: z.boolean().optional(), checkpointInFlight: z.boolean().optional(), + /** A durably claimed discard; no other recovery may act while it is set. */ + discarding: z.string().optional(), operationId: z.string().optional(), messageId: z.string().optional(), stopByMs: z.number().optional(), diff --git a/packages/control-plane/src/session/sandbox-shutdown-safety.test.ts b/packages/control-plane/src/session/sandbox-shutdown-safety.test.ts index 68ca7d78be..e9ee09a3ad 100644 --- a/packages/control-plane/src/session/sandbox-shutdown-safety.test.ts +++ b/packages/control-plane/src/session/sandbox-shutdown-safety.test.ts @@ -186,7 +186,9 @@ describe("sandbox shutdown safety", () => { expect(h.shutdown.admissionDecision()).toBe("held"); await expect(h.shutdown.handleAlarm()).resolves.toBe("hold_watchdogs"); expect(h.stopSandbox).not.toHaveBeenCalled(); - await expect(h.shutdown.recover("retry")).rejects.toThrow("cannot be retried safely"); + // An ordinary checkpoint is not a shutdown: there is no source capture to repeat. + expect(h.shutdown.snapshot()?.availableRecoveryActions).not.toContain("retry"); + await expect(h.shutdown.recover("retry")).rejects.toThrow("Shutdown recovery is unavailable"); await expect(h.shutdown.captureCheckpoint(GENERATION, "execution_complete")).resolves.toEqual( { outcome: "held" } ); diff --git a/packages/control-plane/src/session/sandbox-shutdown.test.ts b/packages/control-plane/src/session/sandbox-shutdown.test.ts index 4e246db51f..cff1f9446f 100644 --- a/packages/control-plane/src/session/sandbox-shutdown.test.ts +++ b/packages/control-plane/src/session/sandbox-shutdown.test.ts @@ -55,6 +55,11 @@ function fixture(providerValue = provider()) { sandboxRow.status = to; return true; }), + discardSandboxState: vi.fn(() => { + sandboxRow.status = "stopped"; + sandboxRow.modal_object_id = null; + return true; + }), }, session: { getSession: vi.fn(() => ({ @@ -79,7 +84,7 @@ function fixture(providerValue = provider()) { getSandboxSocket: vi.fn(() => socket), send: vi.fn(), }, - alarm: { schedule: vi.fn(async () => undefined) }, + alarm: { schedule: vi.fn(async (_atMs: number) => undefined) }, background: { submit: vi.fn((task: () => Promise) => backgroundTasks.push(task)), }, @@ -142,6 +147,15 @@ function reserveGeneration( }); } +/** Every recovery action the public projection offers, including discard. */ +function recoveryActions(shutdown: SandboxShutdownCoordinator): string[] { + const state = shutdown.snapshot(); + return [ + ...(state?.availableRecoveryActions ?? []), + ...(state?.discardAvailable ? ["discard"] : []), + ]; +} + function preparedEvent( state: ShutdownRecord ): Extract { @@ -158,6 +172,93 @@ function preparedEvent( describe("SandboxShutdownCoordinator", () => { beforeEach(() => vi.restoreAllMocks()); + it("commits a VM image before retiring its retained source", async () => { + const takeSnapshot = vi.fn(async () => ({ + success: true as const, + imageId: "vm-image", + sourceStopped: false, + sourceObjectId: "sb-immutable", + })); + const stopSandbox = vi.fn(async () => ({ success: true as const })); + const f = fixture( + provider({ + name: "modal-vm", + capabilities: { ...provider().capabilities, snapshotRequiresShutdown: true }, + takeSnapshot, + stopSandbox, + }) + ); + await readyFinite(f); + await f.shutdown.requestShutdown("execution_complete"); + f.shutdown.prepared(preparedEvent(f.store.value!)); + await f.shutdown.handleAlarm(); + + expect(f.store.value).toMatchObject({ + phase: "saved", + sourceRetired: true, + receipt: { + artifactId: "vm-image", + provider: "modal-vm", + sourceObjectId: "sb-immutable", + }, + }); + expect(f.deps.sandbox.recordSandboxSnapshot).toHaveBeenCalledWith( + GENERATION.sandboxId, + "vm-image", + "runtime-1" + ); + expect(f.deps.sandbox.recordSandboxSnapshot.mock.invocationCallOrder[0]).toBeLessThan( + stopSandbox.mock.invocationCallOrder[0] + ); + expect(stopSandbox).toHaveBeenCalledWith( + expect.objectContaining({ providerObjectId: "sb-immutable" }) + ); + }); + + it("holds a lost VM capture response without retiring the source", async () => { + const takeSnapshot = vi.fn(async () => { + throw new Error("capture response lost"); + }); + const stopSandbox = vi.fn(async () => ({ success: true as const })); + const f = fixture( + provider({ + name: "modal-vm", + capabilities: { ...provider().capabilities, snapshotRequiresShutdown: true }, + takeSnapshot, + stopSandbox, + }) + ); + await readyFinite(f); + await f.shutdown.requestShutdown("inactivity_timeout"); + f.shutdown.prepared(preparedEvent(f.store.value!)); + await f.shutdown.handleAlarm(); + + expect(f.store.value).toMatchObject({ phase: "unknown" }); + expect(stopSandbox).not.toHaveBeenCalled(); + // The source is kept, so it can be captured again. + expect(recoveryActions(f.shutdown)).toEqual(["retry", "discard"]); + }); + + it("does not capture over a checkpoint whose result a restart lost", async () => { + const takeSnapshot = vi.fn(); + const f = fixture(provider({ takeSnapshot })); + await readyFinite(f); + await f.shutdown.requestShutdown("inactivity_timeout"); + f.store.write({ ...f.store.value!, checkpointInFlight: true }); + const restarted = new SandboxShutdownCoordinator(f.deps as never); + + f.setNow(f.store.value!.stopByMs!); + await restarted.handleAlarm(); + + expect(takeSnapshot).not.toHaveBeenCalled(); + expect(f.store.value).toMatchObject({ + phase: "unknown", + checkpointInFlight: true, + error: "An earlier checkpoint has an unknown result.", + }); + expect(recoveryActions(restarted)).not.toContain("retry"); + }); + it("distinguishes unmanaged and held shutdown requests", async () => { const f = fixture(); @@ -442,7 +543,7 @@ describe("SandboxShutdownCoordinator", () => { f.deps.reconcileStatusFromMessages.mockImplementation(async () => { expect(f.deps.failures.record).toHaveBeenCalledWith( "message-1", - "sandbox_lifetime_expiring", + "The sandbox reached its maximum lifetime.", 100_000, "processing" ); @@ -455,7 +556,7 @@ describe("SandboxShutdownCoordinator", () => { expect(f.deps.failures.record).toHaveBeenCalledOnce(); expect(f.deps.failures.record).toHaveBeenCalledWith( "message-1", - "sandbox_lifetime_expiring", + "The sandbox reached its maximum lifetime.", 100_000, "processing" ); @@ -514,7 +615,7 @@ describe("SandboxShutdownCoordinator", () => { expect(await f.shutdown.handleAlarm()).toBe("hold_watchdogs"); expect(f.store.value).toMatchObject({ phase: "unknown", - error: expect.stringContaining("provider result is unknown"), + error: expect.stringContaining("result is unknown"), }); expect(f.deps.provider.takeSnapshot).toBeUndefined(); }); @@ -733,7 +834,7 @@ describe("SandboxShutdownCoordinator", () => { expect(f.store.value).toMatchObject({ phase: "unknown", - error: expect.stringContaining("deadline exceeded"), + error: expect.stringContaining("did not finish before its deadline"), }); expect(stopSandbox).not.toHaveBeenCalled(); } finally { @@ -741,96 +842,399 @@ describe("SandboxShutdownCoordinator", () => { } }); - it("retries only a confirmed pre-capture failure with a new operation", async () => { - const f = fixture( - provider({ - takeSnapshot: vi.fn(async () => ({ - success: true, - imageId: "snapshot-1", - sourceStopped: true, - })), - }) - ); + it("captures without the runtime when it cannot confirm that execution stopped", async () => { + const takeSnapshot = vi.fn(async () => ({ + success: true, + imageId: "snapshot-1", + sourceStopped: true, + })); + const f = fixture(provider({ takeSnapshot })); await readyFinite(f); await f.shutdown.requestShutdown("sandbox_lifetime_expiring"); - const firstOperation = f.store.value!.operationId; + f.backgroundTasks.length = 0; f.shutdown.prepared({ ...preparedEvent(f.store.value!), executionStopped: false, error: "execution_stop_unconfirmed", }); - expect(f.store.value?.phase).toBe("failed"); - f.deps.sockets.send.mockClear(); + await Promise.all(f.backgroundTasks.splice(0).map((task) => task())); + + expect(takeSnapshot).toHaveBeenCalledOnce(); + // The capture cannot prove quiescence, so queued work waits for the user. + expect(f.store.value).toMatchObject({ phase: "saved", continuationPaused: true }); + expect(f.deps.sandbox.updateSandboxStatus).toHaveBeenCalledWith("stale"); + expect(f.calls).toContain("access-retired"); + }); + + it("captures without the runtime when the drain deadline passes", async () => { + const takeSnapshot = vi.fn(async () => ({ + success: true, + imageId: "snapshot-1", + sourceStopped: true, + })); + const f = fixture(provider({ takeSnapshot })); + f.deps.sockets.getSandboxSocket.mockReturnValue(null as never); // An unresponsive runtime. + await readyFinite(f); + await f.shutdown.requestShutdown("inactivity_timeout"); + expect(f.store.value?.phase).toBe("draining"); + + f.setNow(f.store.value!.stopByMs!); + await f.shutdown.handleAlarm(); + + expect(takeSnapshot).toHaveBeenCalledOnce(); + expect(f.store.value).toMatchObject({ + phase: "saved", + reason: "inactivity_timeout", + continuationPaused: true, + }); + }); + + it("retries a failed capture as a new operation", async () => { + const takeSnapshot = vi + .fn>() + .mockRejectedValueOnce(new Error("guest unresponsive")) + .mockResolvedValue({ success: true, imageId: "snapshot-2", sourceStopped: false }); + const stopSandbox = vi.fn(async () => ({ success: true as const })); + const f = fixture(provider({ takeSnapshot, stopSandbox })); + await readyFinite(f); + await f.shutdown.requestShutdown("heartbeat_timeout", "emergency"); + const failed = f.store.value!; + expect(failed).toMatchObject({ phase: "unknown", continuationPaused: true }); + expect(stopSandbox).not.toHaveBeenCalled(); + expect(recoveryActions(f.shutdown)).toEqual(["retry", "discard"]); await f.shutdown.recover("retry"); - expect(f.store.value).toMatchObject({ phase: "draining", error: undefined }); - expect(f.store.value?.operationId).not.toBe(firstOperation); - expect(f.deps.sockets.send).toHaveBeenCalledWith( - expect.anything(), + expect(takeSnapshot).toHaveBeenCalledTimes(2); + expect(f.store.value?.operationId).not.toBe(failed.operationId); + expect(f.store.value).toMatchObject({ + phase: "saved", + continuationPaused: true, + receipt: { artifactId: "snapshot-2" }, + }); + expect(stopSandbox).toHaveBeenCalledWith( + expect.objectContaining({ providerObjectId: "provider-object-1", intent: "destroy" }) + ); + }); + + it("stops offering a retry once the window after the shutdown closes", async () => { + const f = fixture( + provider({ + takeSnapshot: vi.fn(async () => { + throw new Error("guest unresponsive"); + }), + stopSandbox: vi.fn(async () => ({ success: true as const })), + }) + ); + await readyWithoutDeadline(f); + await f.shutdown.requestShutdown("heartbeat_timeout", "emergency"); + const failed = f.store.value!; + + f.setNow(failed.stopByMs! + 30 * 60_000 - 1); + expect(recoveryActions(f.shutdown)).toEqual(["retry", "discard"]); + f.setNow(failed.stopByMs! + 30 * 60_000); + expect(recoveryActions(f.shutdown)).toEqual(["discard"]); + await expect(f.shutdown.recover("retry")).rejects.toThrow("Shutdown recovery is unavailable"); + // The runtime is no longer kept up for a save. + expect(f.shutdown.onRefusedReconnect()).toBe("exit"); + }); + + it("keeps a refused runtime up while its source is being captured", async () => { + let resolveCapture!: (value: { success: true; imageId: string; sourceStopped: true }) => void; + const f = fixture( + provider({ + takeSnapshot: vi.fn( + () => + new Promise<{ success: true; imageId: string; sourceStopped: true }>( + (resolve) => (resolveCapture = resolve) + ) + ), + }) + ); + await readyFinite(f); + expect(f.shutdown.onRefusedReconnect()).toBe("exit"); + + const capture = f.shutdown.requestShutdown("heartbeat_timeout", "emergency"); + await vi.waitFor(() => expect(f.store.value?.phase).toBe("capturing")); + expect(f.shutdown.onRefusedReconnect()).toBe("retry"); + + resolveCapture({ success: true, imageId: "image-1", sourceStopped: true }); + await capture; + expect(f.store.value?.phase).toBe("saved"); + expect(f.shutdown.onRefusedReconnect()).toBe("exit"); + }); + + it("retries a failed save when the runtime reconnects after the capture window", async () => { + const takeSnapshot = vi + .fn>() + .mockRejectedValueOnce(new Error("guest unresponsive")) + .mockResolvedValue({ success: true, imageId: "snapshot-2", sourceStopped: false }); + const stopSandbox = vi.fn(async () => ({ success: true as const })); + const f = fixture(provider({ takeSnapshot, stopSandbox })); + await readyFinite(f); + await f.shutdown.requestShutdown("heartbeat_timeout", "emergency"); + const failed = f.store.value!; + f.backgroundTasks.length = 0; + + // Within the failed attempt's own window the runtime is kept, but not captured again. + expect(f.shutdown.onRefusedReconnect()).toBe("retry"); + expect(f.backgroundTasks).toHaveLength(0); + + f.setNow(failed.captureByMs!); + expect(f.shutdown.onRefusedReconnect()).toBe("retry"); + expect(f.backgroundTasks).toHaveLength(1); + // A second reconnect before the retry runs does not start another capture. + expect(f.shutdown.onRefusedReconnect()).toBe("retry"); + await Promise.all(f.backgroundTasks.splice(0).map((task) => task())); + + expect(takeSnapshot).toHaveBeenCalledTimes(2); + expect(f.store.value).toMatchObject({ phase: "saved", receipt: { artifactId: "snapshot-2" } }); + expect(stopSandbox).toHaveBeenCalledOnce(); + expect(f.shutdown.onRefusedReconnect()).toBe("exit"); + }); + + it("discards a held sandbox by stopping its source and making the next start fresh", async () => { + const stopSandbox = vi.fn(async () => ({ success: true as const })); + const f = fixture( + provider({ + takeSnapshot: vi.fn(async () => { + throw new Error("capture failed"); + }), + stopSandbox, + }) + ); + await readyFinite(f); + await f.shutdown.requestShutdown("heartbeat_timeout", "emergency"); + f.store.write({ + ...f.store.value!, + receipt: { + kind: "snapshot", + artifactId: "older-image", + provider: "modal", + savedAtMs: 500, + runtimeVersion: "runtime-1", + }, + }); + expect(recoveryActions(f.shutdown)).toEqual(["retry", "restore_saved", "discard"]); + // Clients whose schema predates discard still parse the action list. + expect(f.shutdown.snapshot()).toMatchObject({ + availableRecoveryActions: ["retry", "restore_saved"], + discardAvailable: true, + }); + f.deps.background.submit.mockClear(); + + await f.shutdown.recover("discard"); + + expect(stopSandbox).toHaveBeenCalledWith( expect.objectContaining({ - type: "prepare_preservation", - operationId: f.store.value?.operationId, + providerObjectId: "provider-object-1", + reason: "discard", + intent: "destroy", }) ); + expect(f.deps.sandbox.discardSandboxState).toHaveBeenCalledWith(GENERATION); + expect(f.store.value).toMatchObject({ + phase: "running", + providerObjectId: null, + sourceRetired: true, + }); + expect(f.store.value?.receipt).toBeUndefined(); + expect(f.shutdown.snapshot()).toMatchObject({ + phase: "running", + hasRecoveryPoint: false, + availableRecoveryActions: [], + discardAvailable: false, + }); + expect(f.shutdown.isHolding()).toBe(false); + expect(f.shutdown.startupDecision()).toEqual({ kind: "normal" }); + expect(f.shutdown.admissionDecision()).toBe("spawn_required"); + expect(f.deps.background.submit).toHaveBeenCalledWith(expect.any(Function), { + name: "sandbox.lifecycle_change", + }); }); - it("refuses to repeat capture after an unknown provider result", async () => { - const f = fixture(); + it("claims a discard durably so no other recovery can act while its source stops", async () => { + let resolveStop!: (value: { success: true }) => void; + const stopSandbox = vi.fn( + () => new Promise<{ success: true }>((resolve) => (resolveStop = resolve)) + ); + const f = fixture( + provider({ + takeSnapshot: vi.fn(async () => { + throw new Error("capture failed"); + }), + stopSandbox, + }) + ); await readyFinite(f); - await f.shutdown.requestShutdown("sandbox_lifetime_expiring"); - f.store.write({ ...f.store.value!, phase: "unknown", error: "capture outcome unknown" }); - f.deps.provider.takeSnapshot = vi.fn(); + await f.shutdown.requestShutdown("heartbeat_timeout", "emergency"); + f.store.write({ + ...f.store.value!, + receipt: { + kind: "snapshot", + artifactId: "older-image", + provider: "modal", + savedAtMs: 500, + runtimeVersion: "runtime-1", + }, + }); + const operationId = f.store.value!.operationId; - await expect(f.shutdown.recover("retry")).rejects.toThrow( - "unknown provider result cannot be retried" + const discarding = f.shutdown.recover("discard"); + await vi.waitFor(() => expect(stopSandbox).toHaveBeenCalledOnce()); + + expect(f.store.value).toMatchObject({ discarding: expect.any(String), operationId }); + expect(recoveryActions(f.shutdown)).toEqual([]); + await expect(f.shutdown.recover("retry")).rejects.toThrow("Shutdown recovery is unavailable"); + await expect(f.shutdown.recover("restore_saved")).rejects.toThrow( + "Shutdown recovery is unavailable" ); - expect(f.deps.provider.takeSnapshot).not.toHaveBeenCalled(); - expect(f.store.value?.phase).toBe("unknown"); + await expect(f.shutdown.recover("discard")).rejects.toThrow("Shutdown recovery is unavailable"); + expect(f.shutdown.onRefusedReconnect()).toBe("exit"); + + resolveStop({ success: true }); + await discarding; + expect(stopSandbox).toHaveBeenCalledOnce(); + expect(f.store.value).toMatchObject({ phase: "running", providerObjectId: null }); + expect(f.store.value?.discarding).toBeUndefined(); + }); + + it("lets a discard interrupted by a restart be completed, and only completed", async () => { + const stopSandbox = vi.fn(async () => ({ success: true as const })); + const f = fixture(provider({ stopSandbox, takeSnapshot: vi.fn() })); + await readyFinite(f); + f.store.write({ + ...f.store.value!, + phase: "unknown", + error: "capture outcome unknown", + operationId: "failed-capture", + stopByMs: 100_000, + captureByMs: 400_000, + retireByMs: 1_270_000, + discarding: "interrupted-discard", + }); + const restarted = new SandboxShutdownCoordinator(f.deps as never); + + expect(recoveryActions(restarted)).toEqual(["discard"]); + await restarted.recover("discard"); + + expect(stopSandbox).toHaveBeenCalledWith( + expect.objectContaining({ providerObjectId: "provider-object-1", reason: "discard" }) + ); + expect(f.store.value).toMatchObject({ phase: "running", providerObjectId: null }); + }); + + it("keeps the hold when the source cannot be stopped for a discard", async () => { + const stopSandbox = vi.fn(async () => ({ success: false as const, error: "unavailable" })); + const f = fixture( + provider({ + takeSnapshot: vi.fn(async () => { + throw new Error("capture failed"); + }), + stopSandbox, + }) + ); + await readyFinite(f); + await f.shutdown.requestShutdown("heartbeat_timeout", "emergency"); + + await f.shutdown.recover("discard"); + + expect(stopSandbox).toHaveBeenCalledOnce(); + expect(f.deps.sandbox.discardSandboxState).not.toHaveBeenCalled(); + expect(f.store.value).toMatchObject({ + phase: "unknown", + providerObjectId: "provider-object-1", + error: expect.stringContaining("could not be stopped"), + }); + // The claim is released, so every recovery is available again. + expect(f.store.value?.discarding).toBeUndefined(); + expect(recoveryActions(f.shutdown)).toEqual(["retry", "discard"]); + }); + + it("retires this generation's source, not an older receipt's, before restoring it", async () => { + const stopSandbox = vi.fn(async () => ({ success: true as const })); + const f = fixture(provider({ stopSandbox })); + await readyFinite(f); + f.store.write({ + ...f.store.value!, + phase: "unknown", + error: "capture outcome unknown", + receipt: { + kind: "snapshot", + artifactId: "last-good-image", + sourceObjectId: "previous-generation-source", + provider: "modal", + savedAtMs: 500, + runtimeVersion: "runtime-1", + }, + }); + + await f.shutdown.recover("restore_saved"); + + expect(stopSandbox).toHaveBeenCalledOnce(); + expect(stopSandbox).toHaveBeenCalledWith( + expect.objectContaining({ providerObjectId: "provider-object-1" }) + ); + expect(f.store.value).toMatchObject({ + phase: "saved", + receipt: { artifactId: "last-good-image" }, + }); + }); + + it("retries a boot that died after the provider created its sandbox", async () => { + const f = fixture(); + reserveGeneration(f, GENERATION, "confirmed"); + await f.shutdown.recordProviderStartup(GENERATION, { kind: "none", observedAtMs: 100_000 }); + expect(f.store.value).toMatchObject({ providerObjectId: "provider-object-1" }); + expect(f.shutdown.admissionDecision()).toBe("held"); + + for (const status of ["failed", "stale"]) { + f.sandboxRow.status = status; + expect(f.shutdown.admissionDecision()).toBe("spawn_required"); + } + // A runtime that became ready may have served work, so its loss is never a fresh start. + f.shutdown.runtimeReady(1); + expect(f.shutdown.admissionDecision()).toBe("held"); }); it("projects exactly the recovery actions accepted for the current provider and phase", async () => { const f = fixture( provider({ - takeSnapshot: vi.fn(async () => ({ - success: true, - imageId: "snapshot-1", - sourceStopped: true, - })), + takeSnapshot: vi.fn(async () => { + throw new Error("capture failed"); + }), stopSandbox: vi.fn(async () => ({ success: true })), }) ); await readyFinite(f); - await f.shutdown.requestShutdown("sandbox_lifetime_expiring"); - f.shutdown.prepared({ - ...preparedEvent(f.store.value!), - executionStopped: false, - error: "execution_stop_unconfirmed", - }); - expect(f.shutdown.snapshot()?.availableRecoveryActions).toEqual(["retry"]); + await f.shutdown.requestShutdown("heartbeat_timeout", "emergency"); + expect(f.store.value?.phase).toBe("unknown"); + expect(recoveryActions(f.shutdown)).toEqual(["retry", "discard"]); + // A receipt carried from an earlier generation: save again, or go back to it. const receipt = { kind: "snapshot" as const, artifactId: "last-good-image", provider: "modal", - savedAtMs: 50_000, + savedAtMs: 500, runtimeVersion: "runtime-1", }; - f.store.write({ ...f.store.value!, phase: "unknown", receipt }); - expect(f.shutdown.snapshot()?.availableRecoveryActions).toEqual(["restore_saved"]); + f.store.write({ ...f.store.value!, receipt }); + expect(recoveryActions(f.shutdown)).toEqual(["retry", "restore_saved", "discard"]); - f.store.write({ ...f.store.value!, receipt: undefined }); - expect(f.shutdown.snapshot()?.availableRecoveryActions).toEqual([]); + // This generation's source is already captured, so it is restored, not captured again. + f.store.write({ ...f.store.value!, receipt: { ...receipt, savedAtMs: 90_000 } }); + expect(recoveryActions(f.shutdown)).toEqual(["restore_saved", "discard"]); f.store.write({ ...f.store.value!, receipt, provider: "other" }); - expect(f.shutdown.snapshot()?.availableRecoveryActions).toEqual([]); + expect(recoveryActions(f.shutdown)).toEqual([]); f.store.write({ ...f.store.value!, provider: "modal", receipt: { ...receipt, provider: "other" }, }); - expect(f.shutdown.snapshot()?.availableRecoveryActions).toEqual([]); + expect(recoveryActions(f.shutdown)).toEqual([]); await expect(f.shutdown.recover("restore_saved")).rejects.toThrow( "Shutdown recovery is unavailable" ); @@ -853,7 +1257,7 @@ describe("SandboxShutdownCoordinator", () => { }, }); - expect(f.shutdown.snapshot()?.availableRecoveryActions).toEqual([]); + expect(recoveryActions(f.shutdown)).toEqual([]); await expect(f.shutdown.recover("restore_saved")).rejects.toThrow(); expect(f.store.value).toMatchObject({ phase: "saved", continuationPaused: true }); }); @@ -866,6 +1270,13 @@ describe("SandboxShutdownCoordinator", () => { f.sandboxRow.created_at += 1; }, }, + { + name: "discard of a generation that was replaced", + action: "discard" as const, + mutate: (f: ReturnType) => { + f.sandboxRow.created_at += 1; + }, + }, { name: "missing source retirement operation", action: "restore_saved" as const, @@ -892,17 +1303,17 @@ describe("SandboxShutdownCoordinator", () => { }, }, { - name: "legacy lifecycle retry", + name: "retired source", action: "retry" as const, mutate: (f: ReturnType) => { - f.store.write({ ...f.store.value!, lifecyclePolicy: "legacy" }); + f.store.write({ ...f.store.value!, sourceRetired: true }); }, }, { - name: "missing runtime protocol", + name: "retry after the retry window", action: "retry" as const, mutate: (f: ReturnType) => { - f.store.write({ ...f.store.value!, protocolVersion: undefined }); + f.setNow(f.store.value!.stopByMs! + 30 * 60_000); }, }, { @@ -915,24 +1326,18 @@ describe("SandboxShutdownCoordinator", () => { ])("rejects $name without advertising it", async ({ action, mutate }) => { const f = fixture( provider({ - takeSnapshot: vi.fn(async () => ({ - success: true, - imageId: "snapshot-1", - sourceStopped: true, - })), + takeSnapshot: vi.fn(async () => { + throw new Error("capture failed"); + }), }) ); await readyFinite(f); - await f.shutdown.requestShutdown("sandbox_lifetime_expiring"); - f.shutdown.prepared({ - ...preparedEvent(f.store.value!), - executionStopped: false, - error: "execution_stop_unconfirmed", - }); + await f.shutdown.requestShutdown("heartbeat_timeout", "emergency"); + expect(f.store.value?.phase).toBe("unknown"); mutate(f); const before = structuredClone(f.store.value); - expect(f.shutdown.snapshot()?.availableRecoveryActions).not.toContain(action); + expect(recoveryActions(f.shutdown)).not.toContain(action); await expect(f.shutdown.recover(action)).rejects.toThrow("Shutdown recovery is unavailable"); expect(f.store.value).toEqual(before); }); @@ -995,6 +1400,44 @@ describe("SandboxShutdownCoordinator", () => { expect(f.store.value?.sourceRetired).toBe(true); }); + it.each(["discard", "restore_saved"] as const)( + "stops the allocation an interrupted restore created before %s", + async (action) => { + const stopSandbox = vi.fn(async () => ({ success: true as const })); + const f = fixture(provider({ stopSandbox })); + await readyFinite(f); + f.store.write({ + ...f.store.value!, + phase: "saved", + sourceRetired: true, + receipt: { + kind: "snapshot", + artifactId: "saved-image", + provider: "modal", + savedAtMs: 50_000, + runtimeVersion: "runtime-1", + }, + }); + const next = { sandboxId: "sandbox-2", createdAt: 2_000 }; + f.sandboxRow.modal_sandbox_id = next.sandboxId; + f.sandboxRow.created_at = next.createdAt; + reserveGeneration(f, next, "confirmed"); + f.shutdown.markRecoveryInvoked(next); + f.sandboxRow.modal_object_id = "restored-provider-object"; + + // The retirement proof still describes the source the restore replaced. + expect(f.store.value).toMatchObject({ restoreInvoked: true, sourceRetired: true }); + const interrupted = new SandboxShutdownCoordinator(f.deps as never); + expect(recoveryActions(interrupted)).toContain(action); + await interrupted.recover(action); + + expect(stopSandbox).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining({ providerObjectId: "restored-provider-object", intent: "destroy" }) + ); + expect(f.store.value?.phase).toBe(action === "discard" ? "running" : "saved"); + } + ); + it("holds an interrupted snapshot restore until explicit recovery from the retired source", async () => { const f = fixture(); await readyFinite(f); @@ -1035,7 +1478,7 @@ describe("SandboxShutdownCoordinator", () => { providerObjectId: "restored-provider-object", receipt: { artifactId: "saved-image" }, }); - expect(interrupted.snapshot()?.availableRecoveryActions).toEqual(["restore_saved"]); + expect(recoveryActions(interrupted)).toEqual(["restore_saved", "discard"]); await interrupted.recover("restore_saved"); expect(f.store.value).toMatchObject({ phase: "saved", diff --git a/packages/control-plane/src/session/sandbox-shutdown.ts b/packages/control-plane/src/session/sandbox-shutdown.ts index 32d72dcf92..a11eaf1cd1 100644 --- a/packages/control-plane/src/session/sandbox-shutdown.ts +++ b/packages/control-plane/src/session/sandbox-shutdown.ts @@ -32,6 +32,20 @@ const STOP_MS = 60_000; const CAPTURE_MS = 300_000; const RETIRE_MS = 30_000; const MARGIN_MS = 30_000; +/** How long a sandbox whose save failed is kept for another attempt. */ +const RETRY_WINDOW_MS = 30 * 60_000; + +/** User-facing text for a prompt interrupted by a shutdown; reasons are internal codes. */ +const INTERRUPTION_MESSAGES: Record = { + heartbeat_timeout: "The sandbox stopped responding.", + prompt_dispatch_send_failed: "The sandbox stopped responding.", + stop_send_failed: "The sandbox stopped responding.", + stop_alarm_failed: "The sandbox stopped responding.", + stop_confirmation_timeout: "The sandbox stopped responding.", + fatal_runtime_error: "The sandbox runtime failed.", + inactivity_timeout: "The sandbox was stopped after a period of inactivity.", + sandbox_lifetime_expiring: "The sandbox reached its maximum lifetime.", +}; class ShutdownDeadlineError extends Error {} @@ -61,6 +75,7 @@ export class SandboxShutdownCoordinator { private checkpointOperationId: string | null = null; private checkpointGeneration: SandboxGeneration | null = null; private retiringOperation: string | null = null; + private discardingOperation: string | null = null; private activeRestoreGeneration: SandboxGeneration | null = null; private readonly now: () => number; @@ -70,15 +85,20 @@ export class SandboxShutdownCoordinator { snapshot(): SandboxShutdownState | null { const state = this.normalizeInterruptedRestore(); - return state - ? sandboxShutdownSchema.parse({ - ...state, - savedAtMs: state.receipt?.savedAtMs ?? state.savedAtMs, - hasRecoveryPoint: !!state.receipt, - continuationPaused: this.continuationPaused(state), - availableRecoveryActions: this.availableRecoveryActions(state), - }) - : null; + return state ? this.project(state) : null; + } + + /** The public projection; it carries no provider handles or receipts. */ + private project(state: ShutdownRecord): SandboxShutdownState { + const actions = this.availableRecoveryActions(state); + return sandboxShutdownSchema.parse({ + ...state, + savedAtMs: state.receipt?.savedAtMs ?? state.savedAtMs, + hasRecoveryPoint: !!state.receipt, + continuationPaused: this.continuationPaused(state), + availableRecoveryActions: actions.filter((action) => action !== "discard"), + discardAvailable: actions.includes("discard"), + }); } private current(state: ShutdownRecord): boolean { @@ -111,22 +131,15 @@ export class SandboxShutdownCoordinator { this.deps.log?.info("sandbox.preservation", { event: "sandbox.preservation", phase: state.phase, + reason: state.reason, + error: state.error, provider: this.deps.provider.name, sandbox_id: state.generation.sandboxId, generation_created_at: state.generation.createdAt, operation_id: state.operationId, expires_at_ms: state.expiresAtMs, }); - this.broadcast({ - type: "sandbox_preservation", - preservation: sandboxShutdownSchema.parse({ - ...state, - savedAtMs: state.receipt?.savedAtMs ?? state.savedAtMs, - hasRecoveryPoint: !!state.receipt, - continuationPaused: this.continuationPaused(state), - availableRecoveryActions: this.availableRecoveryActions(state), - }), - }); + this.broadcast({ type: "sandbox_preservation", preservation: this.project(state) }); } /** Atomically reserves the sandbox row and shutdown ownership before provider work. */ @@ -288,15 +301,11 @@ export class SandboxShutdownCoordinator { if (state.lifecyclePolicy === "legacy") { return state.checkpointInFlight ? "held" : "ready"; } - // A provider-create failure with no connected runtime/receipt still uses - // the existing fresh-spawn retry policy. Unknown shutdown state never does. - if ( - !state.runtimeReady && - !state.receipt && - !state.providerObjectId && - this.deps.sandbox.getSandbox()?.status === "failed" - ) - return "spawn_required"; + // A runtime that never became ready served no work, so its death is a + // failed boot: the spawn path retries it (restoring any saved state) + // under the circuit breaker. Unknown shutdown state never does. + const row = this.deps.sandbox.getSandbox(); + if (!state.runtimeReady && row && isDeadSandboxStatus(row.status)) return "spawn_required"; if (state.drainAtMs !== null && this.now() >= state.drainAtMs) { this.deps.background.submit(() => this.requestShutdown("sandbox_lifetime_expiring"), { name: "sandbox.preserve", @@ -387,19 +396,18 @@ export class SandboxShutdownCoordinator { async recover(action: ShutdownRecoveryAction): Promise { const state = this.normalizeInterruptedRestore(); if (!state || !this.availableRecoveryActions(state).includes(action)) - throw new ShutdownRecoveryRejectedError( - state?.phase === "unknown" && action === "retry" - ? "An unknown provider result cannot be retried safely; restore a saved recovery point or start a separate session." - : undefined - ); + throw new ShutdownRecoveryRejectedError(); + if (action === "discard") { + await this.discard(state); + return; + } if (state.phase === "saved" && this.continuationPaused(state)) { this.publish({ ...state, continuationPaused: false }); this.notifyLifecycleChange(); return; } if (action === "retry") { - this.publish({ ...state, phase: "running", error: undefined }); - await this.requestShutdown(state.reason ?? "preservation_retry"); + await this.retryCapture(state); return; } const next: ShutdownRecord = { @@ -413,14 +421,17 @@ export class SandboxShutdownCoordinator { }; this.publish(next); if ( - state.sourceRetired || - (state.lifetimeSource === "provider" && - state.expiresAtMs !== null && - this.now() >= state.expiresAtMs) + state.lifetimeSource === "provider" && + state.expiresAtMs !== null && + this.now() >= state.expiresAtMs ) { // The hard provider deadline independently proves the old execution ended. this.finish(next); - } else if (state.providerObjectId) await this.retire(next); + } else if (state.providerObjectId && this.canStopSource()) { + // Stop a known source even when retirement was recorded: an interrupted + // restore keeps the proof about the source it replaced, not its own. + await this.retire(next); + } else if (state.sourceRetired) this.finish(next); else this.fail( next, @@ -436,13 +447,16 @@ export class SandboxShutdownCoordinator { (state.receipt && state.receipt.provider !== this.deps.provider.name) ) return []; - if (state.phase === "failed") { + if (state.phase === "failed" || state.phase === "unknown") { + // A claimed discard can only be completed, and is resubmittable only + // when no call in this instance is still stopping its source. + if (state.discarding) return this.discardingOperation === null ? ["discard"] : []; const actions: ShutdownRecoveryAction[] = []; if (this.canRetryShutdown(state)) actions.push("retry"); if (this.canRestoreSaved(state)) actions.push("restore_saved"); + if (this.canDiscard(state)) actions.push("discard"); return actions; } - if (state.phase === "unknown") return this.canRestoreSaved(state) ? ["restore_saved"] : []; if (state.phase === "saved" && this.continuationPaused(state)) return this.canRestoreSaved(state) ? ["restore_saved"] : []; return []; @@ -456,30 +470,178 @@ export class SandboxShutdownCoordinator { (state.lifetimeSource === "provider" && state.expiresAtMs !== null && this.now() >= state.expiresAtMs) || - (!!state.providerObjectId && - this.deps.provider.capabilities.supportsExplicitStop === true && - !!this.deps.provider.stopSandbox) + (!!state.providerObjectId && this.canStopSource()) ); } + /** + * Another capture of a source that may still hold unsaved work. It needs no + * runtime, so it is offered whether or not the runtime responds. The source + * is kept for it until a fixed time after the shutdown began, which retries + * do not extend. + */ private canRetryShutdown(state: ShutdownRecord): boolean { const provider = this.deps.provider; const canCapture = - (provider.capabilities.supportsPersistentResume === true && - provider.capabilities.supportsExplicitStop === true && - !!provider.stopSandbox) || + (provider.capabilities.supportsPersistentResume === true && this.canStopSource()) || (provider.capabilities.supportsSnapshots === true && !!provider.takeSnapshot); + const now = this.now(); return ( - state.lifecyclePolicy !== "legacy" && - state.protocolVersion === 1 && - state.generationReady && + !state.discarding && + !state.restoreInvoked && !state.checkpointInFlight && + state.sourceRetired !== true && !!state.providerObjectId && + !this.receiptCoversSource(state) && + state.stopByMs !== undefined && + state.captureByMs !== undefined && + now < state.stopByMs + RETRY_WINDOW_MS && canCapture && - (state.expiresAtMs === null || this.now() + RETIRE_MS + MARGIN_MS < state.expiresAtMs) + this.emergencyWindow(state, now).captureByMs > now + MARGIN_MS + ); + } + + /** Captures the held source again under a new operation, without waiting for its runtime. */ + private async retryCapture(state: ShutdownRecord): Promise { + if (this.activeOperation !== null || !this.owns(state) || !this.canRetryShutdown(state)) return; + const next: ShutdownRecord = { + ...state, + error: undefined, + operationId: crypto.randomUUID(), + ...this.emergencyWindow(state, this.now()), + }; + this.fenceRuntime(); + await this.capture(next); + } + + /** + * Nothing may still write to the source once it is discarded, so a known + * source must be stoppable. Recorded retirement is trusted only when it is + * not: it may describe an earlier source than an interrupted restore created. + */ + private canDiscard(state: ShutdownRecord): boolean { + return ( + this.checkpointOperationId === null && + (!state.providerObjectId || this.canStopSource() || state.sourceRetired === true) + ); + } + + private canStopSource(): boolean { + return ( + this.deps.provider.capabilities.supportsExplicitStop === true && + !!this.deps.provider.stopSandbox ); } + /** The receipt was captured from this generation's source rather than carried from an earlier one. */ + private receiptCoversSource(state: ShutdownRecord): boolean { + return !!state.receipt && state.receipt.savedAtMs >= state.generation.createdAt; + } + + /** One absolute budget for a capture that does not wait for the runtime. */ + private emergencyWindow( + state: ShutdownRecord | null, + now: number + ): { captureByMs: number; retireByMs: number } { + const end = Math.min(state?.expiresAtMs ?? Infinity, now + CAPTURE_MS + RETIRE_MS + MARGIN_MS); + return { captureByMs: end - RETIRE_MS - MARGIN_MS, retireByMs: end - MARGIN_MS }; + } + + /** + * A runtime refused at reconnect normally exits, which ends its sandbox. + * While a capture needs that sandbox the runtime is told to retry instead. + * A save that failed while the runtime was unresponsive is attempted again + * now that it is back, at most once per capture window. + */ + onRefusedReconnect(): "retry" | "exit" { + const state = this.deps.store.read(); + if (!state || !this.current(state)) return "exit"; + if (state.phase === "draining" || state.phase === "prepared" || state.phase === "capturing") + return "retry"; + if ((state.phase !== "failed" && state.phase !== "unknown") || !this.canRetryShutdown(state)) + return "exit"; + if (this.activeOperation === null && this.now() >= state.captureByMs!) { + this.deps.log?.info("Retrying a failed save after the runtime reconnected", { + event: "sandbox.preservation_retry", + operation_id: state.operationId, + reason: state.reason, + }); + this.deps.background.submit(() => this.retryCapture(state), { + name: "sandbox.preservation_retry", + }); + } + return "retry"; + } + + /** + * Explicitly abandons unsaved work: stops the source, then leaves a record + * with no receipt and no provider handle so the next start is fresh. + */ + private async discard(state: ShutdownRecord): Promise { + // Claimed durably before any provider I/O, so no other recovery can act on + // this source while it is being stopped, including after a restart. + const claim: ShutdownRecord = { ...state, discarding: crypto.randomUUID(), error: undefined }; + this.discardingOperation = claim.discarding!; + try { + this.publish(claim); + if (claim.providerObjectId && this.canStopSource()) { + try { + await this.stopSource( + claim.providerObjectId, + "discard", + "destroy", + this.now() + RETIRE_MS + ); + } catch { + if (this.ownsDiscard(claim)) { + const error = "The sandbox could not be stopped, so nothing was discarded. Try again."; + this.publish({ ...claim, discarding: undefined, error }); + this.broadcast({ type: "sandbox_warning", message: error }); + } + return; + } + } + if (this.ownsDiscard(claim)) this.completeDiscard(claim); + } finally { + this.discardingOperation = null; + } + } + + private ownsDiscard(claim: ShutdownRecord): boolean { + const current = this.deps.store.read(); + return this.current(claim) && current?.discarding === claim.discarding; + } + + private completeDiscard(state: ShutdownRecord): void { + const next: ShutdownRecord = { + phase: "running", + generation: state.generation, + provider: this.deps.provider.name, + providerObjectId: null, + sourceRetired: true, + lifetimeKind: "unknown", + expiresAtMs: null, + drainAtMs: null, + generationReady: false, + lifecyclePolicy: state.lifecyclePolicy, + }; + this.deps.session.transaction(() => { + if (!this.deps.sandbox.discardSandboxState(state.generation)) + throw new Error("Sandbox generation was superseded"); + this.deps.store.write(next); + }); + this.deps.log?.info("Sandbox discarded", { + event: "sandbox.discarded", + sandbox_id: state.generation.sandboxId, + previous_phase: state.phase, + had_recovery_point: !!state.receipt, + }); + this.announce(next); + this.deps.retireAccess(); + this.broadcast({ type: "sandbox_status", status: "stopped" }); + this.notifyLifecycleChange(); + } + /** Owns an ordinary capture from admission through durable outcome classification. */ async captureCheckpoint( generation: SandboxGeneration, @@ -656,7 +818,14 @@ export class SandboxShutdownCoordinator { } this.deps.store.write(next); if (emergency) this.deps.sandbox.updateSandboxStatus("stale"); - return message ? this.deps.failures.record(message.id, reason, now, "processing") : null; + return message + ? this.deps.failures.record( + message.id, + INTERRUPTION_MESSAGES[reason] ?? "The sandbox was stopped.", + now, + "processing" + ) + : null; }); this.announce(next); if (failure) this.deps.failures.deliver(failure); @@ -685,17 +854,52 @@ export class SandboxShutdownCoordinator { ) return; if (!event.executionStopped || this.now() > state.stopByMs!) { - this.fail( - state, - "failed", - event.error ?? "Active execution did not stop before the graceful shutdown deadline." - ); + const detail = event.error ?? "execution_stop_late"; + this.deps.background.submit(() => this.captureUnconfirmed(state, detail), { + name: "sandbox.preservation_advance", + }); return; } this.publish({ ...state, phase: "prepared" }); // Durable evidence before the critical-event ACK. this.kickAdvance(); } + /** + * The runtime did not confirm that execution stopped, because it is + * unresponsive or could not stop its work. Holding the session would + * preserve nothing, so the source is captured without it. The capture + * cannot prove quiescence, so queued work waits for the user. + */ + private async captureUnconfirmed(state: ShutdownRecord, detail: string): Promise { + if (!this.owns(state)) return; + if (state.checkpointInFlight) { + // A live checkpoint re-drives the drain when it ends. One whose result + // was lost to a restart may still be running at the provider, so no + // capture may race it. + if (!this.checkpointOperationId) + this.fail(state, "unknown", "An earlier checkpoint has an unknown result."); + return; + } + this.deps.log?.warn("Runtime did not confirm shutdown; capturing without it", { + event: "sandbox.preservation_unconfirmed", + operation_id: state.operationId, + reason: state.reason, + detail, + }); + this.fenceRuntime(); + await this.capture({ ...state, continuationPaused: true }); + } + + /** A capture without runtime cooperation first cuts the runtime off from new work. */ + private fenceRuntime(): void { + const row = this.deps.sandbox.getSandbox(); + if (row && !isDeadSandboxStatus(row.status)) { + this.deps.sandbox.updateSandboxStatus("stale"); + this.broadcast({ type: "sandbox_status", status: "stale" }); + } + this.deps.retireAccess(); + } + /** Runs before generic watchdogs, and reasserts the absolute deadline on every alarm. */ async handleAlarm(): Promise<"continue" | "hold_watchdogs"> { const state = this.normalizeInterruptedRestore(); @@ -723,11 +927,7 @@ export class SandboxShutdownCoordinator { if (!this.providerMatches(state)) return; if (state.phase === "draining") { if (this.now() >= state.stopByMs!) { - this.fail( - state, - "failed", - "Could not confirm prompt/tool shutdown before the graceful shutdown deadline." - ); + await this.captureUnconfirmed(state, "stop_deadline_exceeded"); return; } await this.deps.alarm.schedule(state.stopByMs!); @@ -753,7 +953,7 @@ export class SandboxShutdownCoordinator { this.fail( state, "unknown", - "Graceful shutdown was interrupted; the provider result is unknown. No destructive retry was made." + "The save was interrupted by a control-plane restart; its result is unknown." ); return; } @@ -785,7 +985,7 @@ export class SandboxShutdownCoordinator { private async capture(state: ShutdownRecord): Promise { const { provider } = this.deps; if (!state.providerObjectId || this.now() >= state.captureByMs!) { - this.fail(state, "failed", "No time or provider handle remains for a final snapshot."); + this.fail(state, "failed", "No time or provider handle remained to save the sandbox."); return; } this.activeOperation = state.operationId!; @@ -805,6 +1005,7 @@ export class SandboxShutdownCoordinator { }; let artifactId = state.providerObjectId; let sourceStopped = retained; + let sourceObjectId: string | undefined; if (retained) { if (!provider.stopSandbox) throw new Error("Provider cannot preserve-stop this sandbox"); const result = await this.bounded(state.captureByMs!, (signal) => @@ -821,37 +1022,15 @@ export class SandboxShutdownCoordinator { ); artifactId = result.imageId; sourceStopped = result.sourceStopped; + sourceObjectId = result.sourceObjectId; } if (!this.owns(capturing)) return; - const receipt = { - kind: retained ? ("retained" as const) : ("snapshot" as const), + const retiring = this.commitCaptureReceipt( + capturing, artifactId, - provider: provider.name, - savedAtMs: this.now(), - runtimeVersion: this.deps.sandbox.getSandbox()?.runtime_version ?? null, - }; - const retiring: ShutdownRecord = { - ...capturing, - phase: "retiring", - receipt, - savedAtMs: receipt.savedAtMs, - }; - // Receipt and legacy projection describe the same capture. Either both - // commit for this generation or neither may authorize source retirement. - this.deps.session.transaction(() => { - if (!this.owns(capturing)) throw new Error("Snapshot generation was superseded"); - if ( - !retained && - !this.deps.sandbox.recordSandboxSnapshot( - state.generation.sandboxId, - artifactId, - receipt.runtimeVersion - ) - ) - throw new Error("Snapshot generation was superseded"); - this.deps.store.write(retiring); - }); - this.announce(retiring); + retained ? "retained" : "snapshot", + sourceObjectId + ); if (sourceStopped) this.finish(retiring); else await this.retire(retiring); } catch (error) { @@ -860,14 +1039,54 @@ export class SandboxShutdownCoordinator { capturing, "unknown", error instanceof ShutdownDeadlineError - ? "Provider graceful shutdown deadline exceeded; result unknown." - : "The provider did not confirm final graceful shutdown. The previous recovery point is unchanged." + ? "The save did not finish before its deadline; its result is unknown." + : "The provider did not confirm the save. The previous recovery point is unchanged." ); } finally { this.activeOperation = null; } } + private commitCaptureReceipt( + state: ShutdownRecord, + artifactId: string, + kind: "retained" | "snapshot", + sourceObjectId?: string + ): ShutdownRecord { + const receipt = { + kind, + artifactId, + ...(sourceObjectId ? { sourceObjectId } : {}), + provider: this.deps.provider.name, + savedAtMs: this.now(), + runtimeVersion: this.deps.sandbox.getSandbox()?.runtime_version ?? null, + }; + const retiring: ShutdownRecord = { + ...state, + phase: "retiring", + error: undefined, + receipt, + savedAtMs: receipt.savedAtMs, + }; + // Receipt and legacy projection describe the same capture. Either both + // commit for this generation or neither may authorize source retirement. + this.deps.session.transaction(() => { + if (!this.owns(state)) throw new Error("Snapshot generation was superseded"); + if ( + kind === "snapshot" && + !this.deps.sandbox.recordSandboxSnapshot( + state.generation.sandboxId, + artifactId, + receipt.runtimeVersion + ) + ) + throw new Error("Snapshot generation was superseded"); + this.deps.store.write(retiring); + }); + this.announce(retiring); + return retiring; + } + private async retire(state: ShutdownRecord): Promise { if (this.retiringOperation === state.operationId) return; if (!state.receipt || !state.providerObjectId) return; @@ -877,22 +1096,17 @@ export class SandboxShutdownCoordinator { } this.retiringOperation = state.operationId!; try { - if (!this.deps.provider.stopSandbox) - throw new Error("Provider cannot confirm source retirement"); - const session = this.deps.session.getSession()!; - const deadlineAtMs = Math.min(state.retireByMs!, this.now() + RETIRE_MS); - await this.deps.alarm.schedule(deadlineAtMs); - const result = await this.bounded(deadlineAtMs, (signal) => - this.deps.provider.stopSandbox!({ - providerObjectId: state.providerObjectId!, - sessionId: session.session_name || session.id, - reason: state.reason!, - intent: state.receipt!.kind === "snapshot" ? "destroy" : "preserve", - deadlineAtMs, - signal, - }) + await this.deps.alarm.schedule(Math.min(state.retireByMs!, this.now() + RETIRE_MS)); + // A receipt carried from an earlier generation names that generation's + // source; the one to stop is the source this generation is running. + await this.stopSource( + this.receiptCoversSource(state) + ? (state.receipt.sourceObjectId ?? state.providerObjectId) + : state.providerObjectId, + state.reason!, + state.receipt.kind === "snapshot" ? "destroy" : "preserve", + state.retireByMs! ); - if (!result.success) throw new Error(result.error ?? "Source retirement failed"); if (this.owns(state)) this.finish(state); } catch { if (this.owns(state)) @@ -918,10 +1132,34 @@ export class SandboxShutdownCoordinator { this.publish({ ...state, phase, error }); this.broadcast({ type: "sandbox_warning", - message: `Sandbox graceful shutdown ${phase}: ${error}`, + message: `${phase === "failed" ? "Sandbox save failed" : "Sandbox save could not be confirmed"}: ${error}`, }); } + /** Confirmed provider stop of one source, bounded by the caller's deadline. */ + private async stopSource( + providerObjectId: string, + reason: string, + intent: "destroy" | "preserve", + retireByMs: number + ): Promise { + if (!this.deps.provider.stopSandbox) + throw new Error("Provider cannot confirm source retirement"); + const session = this.deps.session.getSession()!; + const deadlineAtMs = Math.min(retireByMs, this.now() + RETIRE_MS); + const result = await this.bounded(deadlineAtMs, (signal) => + this.deps.provider.stopSandbox!({ + providerObjectId, + sessionId: session.session_name || session.id, + reason, + intent, + deadlineAtMs, + signal, + }) + ); + if (!result.success) throw new Error(result.error ?? "Source retirement failed"); + } + private owns(state: ShutdownRecord): boolean { const current = this.deps.store.read(); return ( @@ -997,7 +1235,7 @@ export class SandboxShutdownCoordinator { sessionId: string, reason: string, deadlineAtMs: number - ): Promise<{ imageId: string; sourceStopped: boolean }> { + ): Promise<{ imageId: string; sourceStopped: boolean; sourceObjectId?: string }> { if (!this.deps.provider.takeSnapshot) throw new Error("Provider has no snapshot operation"); const result = await this.bounded(deadlineAtMs, (signal) => this.deps.provider.takeSnapshot!({ @@ -1010,6 +1248,10 @@ export class SandboxShutdownCoordinator { ); if (!result.success || !result.imageId) throw new Error(result.error ?? "Provider snapshot result is unknown"); - return { imageId: result.imageId, sourceStopped: result.sourceStopped === true }; + return { + imageId: result.imageId, + sourceStopped: result.sourceStopped === true, + sourceObjectId: result.sourceObjectId, + }; } } diff --git a/packages/control-plane/src/session/schema.ts b/packages/control-plane/src/session/schema.ts index 2c406465ec..539050da36 100644 --- a/packages/control-plane/src/session/schema.ts +++ b/packages/control-plane/src/session/schema.ts @@ -225,6 +225,7 @@ CREATE TABLE IF NOT EXISTS sandbox ( boot_phase TEXT, -- JSON SandboxBootPhase the runtime last reported; NULL once ready boot_seq INTEGER, -- Sequence of that report, for de-duplicating resends fenced INTEGER NOT NULL DEFAULT 0, -- 1 once the generation's credentials were revoked for good (boot budget) + startup_rejected INTEGER NOT NULL DEFAULT 0, -- rejected startup retains a cleanup obligation created_at INTEGER NOT NULL ); @@ -753,6 +754,11 @@ export const MIGRATIONS: readonly SchemaMigration[] = [ description: "Persist per-step usage in the session", run: STEP_USAGE_TABLE_SQL, }, + { + id: 56, + description: "Retain rejected sandbox startup cleanup intent", + run: "ALTER TABLE sandbox ADD COLUMN startup_rejected INTEGER NOT NULL DEFAULT 0", + }, ]; function removePersistedHookOutputTails(sql: SqlStorage): void { diff --git a/packages/control-plane/src/session/types.ts b/packages/control-plane/src/session/types.ts index d29064454c..bd64c47ebb 100644 --- a/packages/control-plane/src/session/types.ts +++ b/packages/control-plane/src/session/types.ts @@ -213,6 +213,7 @@ export const sandboxRowSchema = z.object({ * cannot self-heal the way a watchdog-failed one may. */ fenced: z.number(), + startup_rejected: z.number().default(0), created_at: z.number(), }); diff --git a/packages/control-plane/src/session/websocket-manager.test.ts b/packages/control-plane/src/session/websocket-manager.test.ts index fcfcadccdc..c340989991 100644 --- a/packages/control-plane/src/session/websocket-manager.test.ts +++ b/packages/control-plane/src/session/websocket-manager.test.ts @@ -208,6 +208,7 @@ function createSandboxRow(modalSandboxId: string): SandboxRow { boot_phase: null, boot_seq: null, fenced: 0, + startup_rejected: 0, created_at: Date.now(), }; } diff --git a/packages/control-plane/test/integration/modal-backend-builds.test.ts b/packages/control-plane/test/integration/modal-backend-builds.test.ts new file mode 100644 index 0000000000..8523064b92 --- /dev/null +++ b/packages/control-plane/test/integration/modal-backend-builds.test.ts @@ -0,0 +1,109 @@ +import { env } from "cloudflare:test"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { createCloudflareEnv } from "../../src/cloudflare/platform"; +import { ImageBuildStore } from "../../src/db/image-builds"; +import { ModalImageBuildAdapter } from "../../src/image-builds/modal-adapter"; +import type { ImageBuildPlannerPort } from "../../src/image-builds/planner"; +import { ImageBuildSessionCleanup } from "../../src/image-builds/session-cleanup"; +import { ImageBuildWorkflow } from "../../src/image-builds/workflow"; +import type { ModalClient } from "../../src/sandbox/client"; +import { ModalSandboxProvider } from "../../src/sandbox/providers/modal-provider"; +import { cleanD1Tables } from "./cleanup"; +import { seedEnvironment, seedImageRowForScope } from "./image-build-helpers"; + +beforeEach(cleanD1Tables); + +describe("Modal backend images over real D1", () => { + it("partitions ready images and build admission by existing provider identity", async () => { + const scope = { kind: "environment", id: await seedEnvironment() } as const; + const store = new ImageBuildStore(env.DB); + for (const provider of ["modal", "modal-vm"] as const) { + await seedImageRowForScope(scope, { + id: provider, + provider, + status: "ready", + providerImageId: "im-" + provider, + }); + expect( + await store.registerBuild({ + id: "build-" + provider, + scope, + provider, + repositoriesFingerprint: "fp", + callbackTokenHash: "hash", + callbackTokenExpiresAt: Date.now() + 60000, + }) + ).toBe(true); + } + for (const provider of ["modal", "modal-vm"] as const) { + expect(await store.getLatestReadyForSpawn(scope, provider)).toMatchObject({ + provider, + provider_image_id: "im-" + provider, + }); + expect(await store.getActiveBuild(scope, provider)).toMatchObject({ + id: "build-" + provider, + }); + } + }); + + it("never starts an incompatible build and retries durable cleanup after reconstruction", async () => { + const scope = { kind: "environment", id: await seedEnvironment() } as const; + const client = { + createImageBuildSandbox: vi + .fn() + .mockResolvedValue({ providerSessionId: "sb-legacy", sandboxBackend: undefined }), + startImageBuildSandbox: vi.fn(), + terminateImageBuildSandbox: vi.fn().mockRejectedValue(new Error("provider unreachable")), + }; + const factory = { + create: () => + new ModalImageBuildAdapter( + new ModalSandboxProvider(client as unknown as ModalClient, "modal-vm") + ), + }; + const planner: ImageBuildPlannerPort = { + resolveTarget: async () => ({ + kind: "environment", + repositories: [{ repoOwner: "acme", repoName: "web", baseBranch: "main" }], + repositoriesFingerprint: "fp", + }), + createCallbackAuth: async () => ({ + token: "token", + tokenHash: "hash", + expiresAt: Date.now() + 60000, + }), + planBuild: async (input) => ({ + ...input, + repositories: input.target.repositories, + repositoriesFingerprint: input.target.repositoriesFingerprint, + buildTimeoutMs: 60000, + callbackToken: "token", + cloneAuth: { type: "unavailable" }, + }), + }; + const correlation = { trace_id: "trace", request_id: "request" }; + const workflow = new ImageBuildWorkflow( + createCloudflareEnv({ ...env, WORKER_URL: "https://worker.test" }), + new ImageBuildStore(env.DB), + factory, + { provider: "modal-vm", planner } + ); + await expect(workflow.triggerBuild(scope, correlation)).rejects.toThrow( + "Failed to trigger build" + ); + expect(client.startImageBuildSandbox).not.toHaveBeenCalled(); + const restartedStore = new ImageBuildStore(env.DB); + const [pending] = await restartedStore.listSessionCleanup(); + expect(pending).toMatchObject({ + provider: "modal-vm", + provider_session_id: "sb-legacy", + provider_session_cleanup_pending: 1, + }); + expect(await restartedStore.getLatestReadyForSpawn(scope, "modal-vm")).toBeNull(); + client.terminateImageBuildSandbox.mockResolvedValue(undefined); + await expect( + new ImageBuildSessionCleanup(restartedStore, factory).run(pending, correlation) + ).resolves.toBe(true); + expect(await restartedStore.listSessionCleanup()).toEqual([]); + }); +}); diff --git a/packages/control-plane/test/integration/sandbox-shutdown.test.ts b/packages/control-plane/test/integration/sandbox-shutdown.test.ts index 1c5551e3f5..ad69b252c8 100644 --- a/packages/control-plane/test/integration/sandbox-shutdown.test.ts +++ b/packages/control-plane/test/integration/sandbox-shutdown.test.ts @@ -77,6 +77,170 @@ async function readShutdown(stub: DurableObjectStub): Promise { + it("holds an interrupted legacy VM capture without recapture or retirement", async () => { + const { stub } = await initNamedSession(`vm-capture-receipt-${Date.now()}`); + await seedSandboxAuth(stub, { authToken: AUTH_TOKEN, sandboxId: SANDBOX_ID, status: "ready" }); + await runInSessionDO(stub, (_instance, state) => { + state.storage.sql.exec("UPDATE sandbox SET modal_object_id = 'sb-captured'"); + }); + const now = Date.now(); + await seedShutdown(stub, { + phase: "capturing", + provider: "modal-vm", + providerObjectId: "sb-captured", + operationId: "lost-terminal-capture", + captureReceiptPending: true, + stopByMs: now - 120_000, + captureByMs: now - 60_000, + retireByMs: now - 30_000, + generationReady: true, + lifecyclePolicy: "confirmed", + protocolVersion: 1, + }); + const evidence = await runInSessionDO(stub, async (instance, durableState) => { + let captureCount = 0; + let stopCount = 0; + const provider: SandboxProvider = { + name: "modal-vm", + capabilities: { + supportsSandboxTimeout: true, + supportsSnapshots: true, + snapshotRequiresShutdown: true, + supportsRestore: true, + supportsExplicitStop: true, + }, + createSandbox: async () => { + throw new Error("must not create"); + }, + takeSnapshot: async () => { + captureCount++; + throw new Error("must not recapture"); + }, + stopSandbox: async () => { + stopCount++; + return { success: true }; + }, + }; + const restarted = realLifecycleHarness(instance, durableState, provider); + await restarted.manager.handleShutdownAlarm(); + return { captureCount, stopCount, snapshot: restarted.manager.shutdownSnapshot() }; + }); + expect(evidence).toMatchObject({ + captureCount: 0, + stopCount: 0, + snapshot: { phase: "unknown", hasRecoveryPoint: false }, + }); + expect(await queryDO(stub, "SELECT snapshot_image_id FROM sandbox")).toEqual([ + { snapshot_image_id: null }, + ]); + }); + + it("discards a held VM through the lifecycle boundary so the next start is fresh", async () => { + const { stub } = await initNamedSession(`vm-discard-${Date.now()}`); + await seedSandboxAuth(stub, { authToken: AUTH_TOKEN, sandboxId: SANDBOX_ID, status: "stale" }); + await runInSessionDO(stub, (_instance, state) => { + state.storage.sql.exec( + `UPDATE sandbox SET modal_object_id = 'sb-held', snapshot_image_id = 'im-older', + snapshot_runtime_version = 'v1'` + ); + }); + const now = Date.now(); + await seedShutdown(stub, { + phase: "unknown", + provider: "modal-vm", + providerObjectId: "sb-held", + error: "The provider did not confirm the save.", + reason: "heartbeat_timeout", + operationId: "failed-capture", + stopByMs: now - 60_000, + captureByMs: now + 240_000, + retireByMs: now + 270_000, + continuationPaused: true, + generationReady: true, + runtimeReady: true, + lifecyclePolicy: "confirmed", + protocolVersion: 1, + receipt: { + kind: "snapshot", + artifactId: "im-older", + provider: "modal-vm", + savedAtMs: 1, + runtimeVersion: "v1", + }, + }); + const evidence = await runInSessionDO(stub, async (instance, durableState) => { + const stopped: string[] = []; + const provider: SandboxProvider = { + name: "modal-vm", + capabilities: { + supportsSandboxTimeout: true, + supportsSnapshots: true, + snapshotRequiresShutdown: true, + supportsRestore: true, + supportsExplicitStop: true, + }, + createSandbox: async () => { + throw new Error("must not create during discard"); + }, + stopSandbox: async ({ providerObjectId }) => { + stopped.push(providerObjectId); + return { success: true }; + }, + }; + const restarted = realLifecycleHarness(instance, durableState, provider); + const before = restarted.manager.shutdownSnapshot(); + await restarted.manager.recoverShutdown("discard"); + return { + before, + stopped, + after: restarted.manager.shutdownSnapshot(), + admission: restarted.manager.pushAdmissionDecision(), + }; + }); + expect(evidence.before).toMatchObject({ phase: "unknown", discardAvailable: true }); + expect(evidence).toMatchObject({ + stopped: ["sb-held"], + after: { phase: "running", hasRecoveryPoint: false, discardAvailable: false }, + admission: "start_required", + }); + expect( + await queryDO(stub, "SELECT status, snapshot_image_id, modal_object_id FROM sandbox") + ).toEqual([{ status: "stopped", snapshot_image_id: null, modal_object_id: null }]); + }); + + it("saves an archived session's sandbox and keeps it through a reconnect during the save", async () => { + const name = `archive-preserves-${Date.now()}`; + const { stub } = await initNamedSession(name); + await seedSandboxAuth(stub, { authToken: AUTH_TOKEN, sandboxId: SANDBOX_ID, status: "ready" }); + await runInSessionDO(stub, (_instance, state) => { + state.storage.sql.exec("UPDATE sandbox SET modal_object_id = 'sb-live'"); + }); + await seedShutdown(stub, { + providerObjectId: "sb-live", + generationReady: true, + runtimeReady: true, + protocolVersion: 1, + lifecyclePolicy: "confirmed", + }); + await queryDO(stub, "UPDATE session SET status = 'completed'"); + + const archived = await stub.fetch("http://internal/internal/archive", { method: "POST" }); + expect(archived.status).toBe(200); + expect(await readShutdown(stub)).toMatchObject({ + phase: "draining", + reason: "session_archived", + }); + + // Archive refuses the runtime's reconnect; the save still needs its sandbox. + const { ws, response } = await openSandboxWs(name, { + authToken: AUTH_TOKEN, + sandboxId: SANDBOX_ID, + }); + expect(ws).toBeNull(); + expect(response.status).toBe(503); + expect(await response.text()).toBe("Sandbox is being saved"); + }); + it("preserves a completed session status when shutdown begins between prompts", async () => { const name = `shutdown-completed-status-${Date.now()}`; const { stub } = await initNamedSession(name); @@ -476,7 +640,7 @@ describe("sandbox graceful shutdown wiring", () => { supportsRestore: true, supportsExplicitStop: true, supportsPersistentResume: false, - snapshotStopsSandbox: true, + snapshotRequiresShutdown: true, }, createSandbox: async () => { throw new Error("not used by inactivity regression"); diff --git a/packages/modal-infra/README.md b/packages/modal-infra/README.md index 3ee16bf6b7..9efcdb796e 100644 --- a/packages/modal-infra/README.md +++ b/packages/modal-infra/README.md @@ -110,6 +110,8 @@ pip install -e ".[dev]" ```bash # Build the dynamic Sandbox image, then deploy the app (recommended) uv run python deploy.py --build-sandbox-image +# Also build and verify the Docker-capable image variant (docs/MODAL_DOCKER.md): +# BUILD_MODAL_VM_IMAGE=true uv run python deploy.py --build-sandbox-image uv run modal deploy deploy.py # Alternative app deployment after the same eager image-build step diff --git a/packages/modal-infra/deploy.py b/packages/modal-infra/deploy.py index 8261a6ae58..2d1e979cfd 100644 --- a/packages/modal-infra/deploy.py +++ b/packages/modal-infra/deploy.py @@ -10,6 +10,7 @@ import argparse import json +import os import sys from pathlib import Path @@ -22,16 +23,66 @@ # The eager builder must run before a verified image reference exists. Import # only build modules, without src.__init__ registering deployable functions. from app_config import APP_NAME - from images.base import base_image, base_image_plan, image_reference_path + from images.base import ( + MODAL_VM_EXPERIMENTAL_OPTIONS, + base_image, + base_image_plan, + docker_image, + image_reference_path, + ) + from sandbox.launch_policy import VM_DEFAULT_CPU_CORES, VM_DEFAULT_MEMORY_MIB else: # Modal imports this module to discover the fully registered application. from src.app import app from src.app_config import APP_NAME - from src.images.base import base_image, base_image_plan, image_reference_path + from src.images.base import ( + MODAL_VM_EXPERIMENTAL_OPTIONS, + base_image, + base_image_plan, + docker_image, + image_reference_path, + ) + from src.sandbox.launch_policy import VM_DEFAULT_CPU_CORES, VM_DEFAULT_MEMORY_MIB + +# Resources the Docker image is verified with; the same defaults the control +# plane freezes into a Docker session that configures none. +DOCKER_VERIFICATION_CPU_CORES = VM_DEFAULT_CPU_CORES +DOCKER_VERIFICATION_MEMORY_MIB = VM_DEFAULT_MEMORY_MIB + + +def _verify_image(sandbox: modal.Sandbox, *scripts: tuple[str, ...]) -> None: + for script in scripts: + process = sandbox.exec("/opt/openinspect/python/bin/python", *script, timeout=240) + process.stdout.read() + process.wait() + if process.returncode != 0: + raise RuntimeError(f"Modal image verification failed: {process.stderr.read()}") -def build_sandbox_image() -> None: - """Build the image used by dynamic sandboxes before requests can create them.""" +def _publish_image_record(record: dict[str, str]) -> None: + path = image_reference_path() + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(record) + "\n") + + +def _deployed_vm_image() -> str | None: + """Retain the existing VM capability until the consumer selector has switched.""" + try: + image_id = modal.Function.from_name(APP_NAME, "deployment_vm_image").remote() + except modal.exception.NotFoundError: + # First deployment, or a pre-VM app. Other failures must block deployment. + return None + if image_id is not None and (not isinstance(image_id, str) or not image_id.strip()): + raise RuntimeError("Deployed VM image reference is invalid") + return image_id + + +def build_sandbox_image(*, with_docker: bool = False) -> None: + """Build the image used by dynamic sandboxes before requests can create them. + + The default image is verified and published first, so a failed Docker + variant never replaces a known-good default reference. + """ if base_image_plan is None: raise RuntimeError("Modal sandbox image build requires a local packed image plan") deployed_app = modal.App.lookup(APP_NAME, create_if_missing=True) @@ -52,16 +103,7 @@ def build_sandbox_image() -> None: timeout=300, ) try: - process = sandbox.exec( - "/opt/openinspect/python/bin/python", - "/app/verify/smoke_test.py", - "verify", - timeout=240, - ) - process.stdout.read() - process.wait() - if process.returncode != 0: - raise RuntimeError(f"Modal image verification failed: {process.stderr.read()}") + _verify_image(sandbox, ("/app/verify/smoke_test.py", "verify")) write_build_result(base_image.object_id) finally: sandbox.terminate() @@ -70,9 +112,40 @@ def build_sandbox_image() -> None: "imageId": base_image.object_id, "buildHash": base_image_plan["buildHash"], } - path = image_reference_path() - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(record) + "\n") + if not with_docker: + retained_vm_image = _deployed_vm_image() + if retained_vm_image: + record["dockerImageId"] = retained_vm_image + _publish_image_record(record) + if not with_docker: + return + + if docker_image is None: + raise RuntimeError("Docker sandbox image build requires a local packed image plan") + with modal.enable_output(): + docker_image.build(deployed_app) + # Verify on the VM runtime the variant is launched with, running the + # standard smoke suite plus a real daemon, build, run and Compose check. + sandbox = modal.Sandbox.create( + "sleep", + "infinity", + app=deployed_app, + image=modal.Image.from_id(docker_image.object_id), + env=base_image_plan["runtimeEnv"], + timeout=600, + cpu=DOCKER_VERIFICATION_CPU_CORES, + memory=DOCKER_VERIFICATION_MEMORY_MIB, + experimental_options=dict(MODAL_VM_EXPERIMENTAL_OPTIONS), + ) + try: + _verify_image( + sandbox, + ("/app/verify/smoke_test.py", "verify"), + ("/app/verify/docker_smoke.py",), + ) + finally: + sandbox.terminate() + _publish_image_record({**record, "dockerImageId": docker_image.object_id}) def main() -> None: @@ -80,7 +153,8 @@ def main() -> None: parser.add_argument("--build-sandbox-image", action="store_true") args = parser.parse_args() if args.build_sandbox_image: - build_sandbox_image() + # BUILD_MODAL_VM_IMAGE is the same switch Terraform and images/base.py read. + build_sandbox_image(with_docker=os.environ.get("BUILD_MODAL_VM_IMAGE") == "true") if __name__ == "__main__": diff --git a/packages/modal-infra/src/images/base.py b/packages/modal-infra/src/images/base.py index b5c7f09e51..8ec8d28d2a 100644 --- a/packages/modal-infra/src/images/base.py +++ b/packages/modal-infra/src/images/base.py @@ -14,6 +14,11 @@ CACHE_BUSTER = RUNTIME_VERSION IMAGE_ID_ENV = "OPENINSPECT_MODAL_BASE_IMAGE_ID" +# The optional Docker-capable variant: the default image plus one install phase. +# Provisioned separately; absent until an operator builds and verifies it. +DOCKER_IMAGE_ID_ENV = "OPENINSPECT_MODAL_DOCKER_IMAGE_ID" +# The Modal option that launches a Docker-capable VM instead of a gVisor sandbox. +MODAL_VM_EXPERIMENTAL_OPTIONS: dict[str, bool] = {"vm_runtime": True} def local_image_plan() -> tuple[Path, dict[str, Any]]: @@ -36,7 +41,11 @@ def deployed_image_environment() -> dict[str, str]: image_id = os.environ.get(IMAGE_ID_ENV) if not image_id: raise RuntimeError("Deployed Modal function is missing its verified sandbox image ID") - return {IMAGE_ID_ENV: image_id} + environment = {IMAGE_ID_ENV: image_id} + docker_image_id = os.environ.get(DOCKER_IMAGE_ID_ENV) + if docker_image_id: + environment[DOCKER_IMAGE_ID_ENV] = docker_image_id + return environment path = image_reference_path() if not path.is_file(): raise RuntimeError("Build the Modal sandbox image before deploying functions") @@ -47,7 +56,15 @@ def deployed_image_environment() -> dict[str, str]: image_id = record.get("imageId") if not isinstance(image_id, str) or not image_id.strip(): raise RuntimeError("Built Modal image record is missing its verified sandbox image ID") - return {IMAGE_ID_ENV: image_id} + environment = {IMAGE_ID_ENV: image_id} + docker_image_id = record.get("dockerImageId") + if docker_image_id is not None: + if not isinstance(docker_image_id, str) or not docker_image_id.strip(): + raise RuntimeError("Built Modal image record has an invalid Docker image ID") + environment[DOCKER_IMAGE_ID_ENV] = docker_image_id + if os.environ.get("BUILD_MODAL_VM_IMAGE") == "true" and DOCKER_IMAGE_ID_ENV not in environment: + raise RuntimeError("Build and verify the Docker sandbox image before deploying functions") + return environment def _define_image() -> tuple[modal.Image, dict[str, Any] | None]: @@ -68,3 +85,16 @@ def _define_image() -> tuple[modal.Image, dict[str, Any] | None]: base_image, base_image_plan = _define_image() + + +def _define_docker_image() -> modal.Image | None: + if not modal.is_local(): + image_id = os.environ.get(DOCKER_IMAGE_ID_ENV) + return modal.Image.from_id(image_id) if image_id else None + # Same bundle, one extra phase; the default image layers stay untouched. + return base_image.run_commands( + "bash /tmp/openinspect-image/packages/sandbox-images/install/install.sh docker" + ) + + +docker_image = _define_docker_image() diff --git a/packages/modal-infra/src/sandbox/build_session.py b/packages/modal-infra/src/sandbox/build_session.py index 2dfd467be3..b8a4a8e596 100644 --- a/packages/modal-infra/src/sandbox/build_session.py +++ b/packages/modal-infra/src/sandbox/build_session.py @@ -2,7 +2,8 @@ import json import time -from typing import cast +from dataclasses import dataclass +from typing import Any, cast import modal @@ -22,7 +23,16 @@ ) from ..app import app +from ..app_config import APP_NAME from ..images.base import base_image +from .launch_policy import ( + ModalBackend, + _identity_digest, + docker_base_image, + docker_runtime_env, + launch_kwargs, + parse_launch, +) from .manager import SNAPSHOT_FILESYSTEM_TIMEOUT_SECONDS from .vcs_env import inject_vcs_env_vars @@ -52,6 +62,12 @@ class BuildSessionNotFoundError(LookupError): """The requested provider session is absent or bound to another build.""" +@dataclass(frozen=True) +class BuildSessionLaunch: + provider_session_id: str + sandbox_backend: ModalBackend + + class ModalBuildSessionService: """Own the identity-bound lifecycle of one Modal image-build sandbox.""" @@ -70,8 +86,13 @@ async def create( user_env_vars: dict[str, str] | None = None, build_execution_timeout_seconds: int = DEFAULT_BUILD_TIMEOUT_SECONDS, timeout_seconds: int = DEFAULT_BUILD_TIMEOUT_SECONDS, - ) -> str: + sandbox_settings: dict[str, Any] | None = None, + sandbox_backend: ModalBackend = "modal", + ) -> BuildSessionLaunch: start_time = time.time() + docker = parse_launch(sandbox_backend, sandbox_settings) + if sandbox_backend == "modal": + docker = parse_launch(sandbox_backend, None) # Preserve standard build sizing. primary = repositories[0] env_vars = dict(user_env_vars or {}) for name in RESERVED_USER_ENV_KEYS: @@ -93,6 +114,7 @@ async def create( BUILD_ID_ENV: build_id, CALLBACK_URL_ENV: callback_url, FAILURE_CALLBACK_URL_ENV: failure_callback_url, + **docker_runtime_env(docker), } ) inject_vcs_env_vars( @@ -109,18 +131,31 @@ async def create( "openinspect_scope_kind": scope_kind, "openinspect_scope_id": scope_id, LAUNCH_PROTOCOL_TAG: MODAL_IMAGE_BUILD_START_PROTOCOL, + "openinspect_backend": sandbox_backend, } - sandbox = await modal.Sandbox.create.aio( - *command, - image=base_image, - app=app, - secrets=[], - timeout=timeout_seconds, - workdir="/workspace", - env=cast("dict[str, str | None]", env_vars), - tags=tags, - ) + name = self._allocation_name(build_id, sandbox_backend) + sandbox = await self._find(build_id, sandbox_backend) + if sandbox is not None and await sandbox.get_tags.aio() != tags: + raise RuntimeError("Build allocation ownership mismatch") + if sandbox is None: + try: + sandbox = await modal.Sandbox.create.aio( + *command, + image=docker_base_image() if docker.enabled else base_image, + app=app, + secrets=[], + timeout=timeout_seconds, + workdir="/workspace", + env=cast("dict[str, str | None]", env_vars), + tags=tags, + name=name, + **launch_kwargs(docker), + ) + except modal.exception.AlreadyExistsError: + sandbox = await self._find(build_id, sandbox_backend) + if sandbox is None or await sandbox.get_tags.aio() != tags: + raise RuntimeError("Build allocation ownership mismatch") from None log.info( "sandbox.create_build", build_id=build_id, @@ -130,7 +165,30 @@ async def create( duration_ms=int((time.time() - start_time) * 1000), outcome="success", ) - return sandbox.object_id + return BuildSessionLaunch( + provider_session_id=sandbox.object_id, sandbox_backend=docker.backend + ) + + @staticmethod + def _allocation_name(build_id: str, backend: ModalBackend) -> str: + return "oi-build-" + _identity_digest(backend, build_id)[:40] + + @classmethod + async def _find(cls, build_id: str, backend: ModalBackend) -> modal.Sandbox | None: + try: + sandbox = await modal.Sandbox.from_name.aio( + APP_NAME, cls._allocation_name(build_id, backend) + ) + except modal.exception.NotFoundError: + return None + tags = await sandbox.get_tags.aio() + if ( + tags.get("openinspect_kind") != "image-build" + or tags.get("openinspect_build_id") != build_id + or tags.get("openinspect_backend") != backend + ): + raise RuntimeError("Build allocation ownership mismatch") + return sandbox async def start( self, diff --git a/packages/modal-infra/src/sandbox/launch_policy.py b/packages/modal-infra/src/sandbox/launch_policy.py new file mode 100644 index 0000000000..6bcfb210ba --- /dev/null +++ b/packages/modal-infra/src/sandbox/launch_policy.py @@ -0,0 +1,120 @@ +"""Modal compute offerings: launch resources, runtime options, and allocation identity.""" + +from __future__ import annotations + +import hashlib +import math +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Literal + +from sandbox_runtime.constants import DOCKER_ENABLED_ENV_VAR + +if TYPE_CHECKING: + import modal + +ALLOCATION_NAME_PREFIX = "oi-" +ALLOCATION_KIND_TAG = "openinspect_kind" +ALLOCATION_SESSION_TAG = "openinspect_session_id" +ALLOCATION_SANDBOX_TAG = "openinspect_sandbox_id" +ALLOCATION_BACKEND_TAG = "openinspect_backend" +ModalBackend = Literal["modal", "modal-vm"] +VM_DEFAULT_CPU_CORES = 2 +VM_DEFAULT_MEMORY_MIB = 4096 + + +class InvalidDockerSettingsError(ValueError): + """The Docker-sensitive settings are malformed or contradictory.""" + + +class DockerImageUnavailableError(RuntimeError): + """The deployment has no verified Docker-capable sandbox image.""" + + +@dataclass(frozen=True) +class ModalLaunch: + backend: ModalBackend + cpu_cores: float | None = None + memory_mib: int | None = None + + @property + def enabled(self) -> bool: + return self.backend == "modal-vm" + + +def parse_launch(backend: ModalBackend, settings: dict[str, Any] | None) -> ModalLaunch: + """Select an offering independently of generic resource settings.""" + if backend not in ("modal", "modal-vm"): + raise InvalidDockerSettingsError("Unknown Modal sandbox backend") + settings = settings or {} + if "dockerEnabled" in settings: + raise InvalidDockerSettingsError( + "dockerEnabled was removed; select SANDBOX_PROVIDER=modal-vm" + ) + cpu_cores = settings.get("cpuCores") + memory_mib = settings.get("memoryMib") + if backend == "modal-vm": + cpu_cores = VM_DEFAULT_CPU_CORES if cpu_cores is None else cpu_cores + memory_mib = VM_DEFAULT_MEMORY_MIB if memory_mib is None else memory_mib + if cpu_cores is not None and ( + isinstance(cpu_cores, bool) + or not isinstance(cpu_cores, int | float) + or not math.isfinite(cpu_cores) + or cpu_cores <= 0 + ): + raise InvalidDockerSettingsError("cpuCores must be positive and finite") + if memory_mib is not None and ( + isinstance(memory_mib, bool) or not isinstance(memory_mib, int) or memory_mib <= 0 + ): + raise InvalidDockerSettingsError("memoryMib must be a positive integer") + return ModalLaunch(backend=backend, cpu_cores=cpu_cores, memory_mib=memory_mib) + + +def docker_base_image() -> modal.Image: + """The verified Docker-capable base image; never the default image.""" + from ..images.base import docker_image + + if docker_image is None: + raise DockerImageUnavailableError("Docker sandbox image is not provisioned") + return docker_image + + +def launch_kwargs(launch: ModalLaunch) -> dict[str, Any]: + """One mapping for outer allocation resources and VM runtime selection.""" + result: dict[str, Any] = {} + if launch.enabled: + result["experimental_options"] = {"vm_runtime": True} + if launch.cpu_cores is not None: + # Agent-controlled VM workloads must not burst beyond their CPU request. + result["cpu"] = (launch.cpu_cores, launch.cpu_cores) if launch.enabled else launch.cpu_cores + if launch.memory_mib is not None: + result["memory"] = launch.memory_mib + return result + + +def docker_runtime_env(launch: ModalLaunch) -> dict[str, str]: + """The trusted runtime signal, always set explicitly by the provider.""" + return {DOCKER_ENABLED_ENV_VAR: "true" if launch.enabled else "false"} + + +def _identity_digest(*parts: str) -> str: + return hashlib.sha256("\n".join(parts).encode()).hexdigest() + + +def docker_allocation_name(session_id: str) -> str: + """One provider-enforced running allocation slot per session. + + Generations retain distinct ownership tags, not distinct names. A missing + lookup cannot authorize overlapping creates: Modal rejects a conflicting + name until the previous sandbox has completely stopped. + """ + return ALLOCATION_NAME_PREFIX + _identity_digest("modal-vm", session_id)[:40] + + +def docker_allocation_tags(session_id: str, sandbox_id: str) -> dict[str, str]: + """Ownership tags a found allocation must match exactly before adoption or retirement.""" + return { + ALLOCATION_KIND_TAG: "session", + ALLOCATION_SESSION_TAG: _identity_digest(session_id)[:48], + ALLOCATION_SANDBOX_TAG: _identity_digest(sandbox_id)[:48], + ALLOCATION_BACKEND_TAG: "modal-vm", + } diff --git a/packages/modal-infra/src/sandbox/manager.py b/packages/modal-infra/src/sandbox/manager.py index 727b61fc97..dca92f935b 100644 --- a/packages/modal-infra/src/sandbox/manager.py +++ b/packages/modal-infra/src/sandbox/manager.py @@ -21,6 +21,7 @@ CODE_SERVER_PORT, CODE_SERVER_PORT_ENV_VAR, DEFAULT_SANDBOX_TIMEOUT_SECONDS, + DOCKER_ENABLED_ENV_VAR, EXPECTED_TUNNEL_PORTS_ENV_VAR, NOVNC_PORT, NOVNC_PORT_ENV_VAR, @@ -33,16 +34,28 @@ VNC_PASSWORD_MAX_BYTES, VNC_PORT, ) +from sandbox_runtime.docker_control import CONTROL_TIMEOUT_SECONDS from sandbox_runtime.log_config import get_logger from sandbox_runtime.types import SandboxStatus, SessionConfig from ..app import app +from ..app_config import APP_NAME from ..images.base import base_image +from .launch_policy import ( + ModalBackend, + docker_allocation_name, + docker_allocation_tags, + docker_base_image, + docker_runtime_env, + launch_kwargs, + parse_launch, +) from .vcs_env import inject_vcs_env_vars log = get_logger("manager") SNAPSHOT_FILESYSTEM_TIMEOUT_SECONDS = 300 +ACCESS_PASSWORD_READ_TIMEOUT_SECONDS = 30 MAX_TUNNEL_PORTS = 10 DEFAULT_VNC_ENABLED = False _RESERVED_LAUNCH_ENV_VARS = { @@ -55,6 +68,7 @@ "SESSION_CONFIG", VNC_PASSWORD_ENV_VAR, NOVNC_PORT_ENV_VAR, + DOCKER_ENABLED_ENV_VAR, } @@ -70,27 +84,32 @@ def _has_repository(repo_owner: str | None, repo_name: str | None) -> bool: return has_owner -def _resource_kwargs(settings: dict[str, Any] | None) -> dict: - """Map sandbox settings to Modal resource kwargs. - - `cpuCores` -> Modal `cpu` (cores, fractional allowed), `memoryMib` -> Modal - `memory` (MiB). The control plane owns normalization; this only maps - already-normalized settings into provider-specific argument names. - """ - if not settings: - return {} - - kwargs: dict = {} - - cpu_cores = settings.get("cpuCores") - if cpu_cores is not None: - kwargs["cpu"] = float(cpu_cores) +async def _create_sandbox( + create_kwargs: dict[str, Any], *, repository_image: bool +) -> modal.Sandbox: + """The one `Sandbox.create` call; only its own NotFound means the image is gone.""" + try: + return await modal.Sandbox.create.aio( + "python", + "-m", + "sandbox_runtime.entrypoint", + **create_kwargs, + ) + except modal.exception.NotFoundError as e: + if repository_image: + raise RepositoryImageUnavailableError("repository image is unavailable") from e + raise - memory_mib = settings.get("memoryMib") - if memory_mib is not None: - kwargs["memory"] = memory_mib - return kwargs +def _session_identity(session_config: SessionConfig | dict[str, Any] | None) -> str: + """The control-plane session id carried in the launch's session config.""" + if isinstance(session_config, dict): + session_id = session_config.get("session_id") + elif session_config is not None: + session_id = session_config.session_id + else: + session_id = None + return session_id if isinstance(session_id, str) else "" @dataclass @@ -99,6 +118,7 @@ class SandboxConfig: repo_owner: str | None repo_name: str | None + sandbox_backend: ModalBackend = "modal" sandbox_id: str | None = None # Expected sandbox ID from control plane session_config: SessionConfig | dict[str, Any] | None = None control_plane_url: str = "" @@ -115,6 +135,10 @@ class SandboxConfig: settings: dict[str, Any] | None = ( None # Sandbox settings (tunnelPorts, etc.) from control plane ) + # A previous generation's sandbox id whose Docker VM may still be running + # after an ambiguous create (the control plane lost the response). Only + # Docker launches act on it; the named allocation is retired if owned. + retire_sandbox_id: str | None = None @dataclass @@ -133,6 +157,7 @@ class SandboxHandle: vnc_password: str | None = None ttyd_url: str | None = None # proxy tunnel URL (not ttyd directly) tunnel_urls: dict[int, str] | None = None # port -> tunnel URL mapping for extra ports + sandbox_backend: ModalBackend = "modal" @dataclass(frozen=True) @@ -368,6 +393,7 @@ async def _launch_sandbox(self, spec: _SandboxLaunchSpec) -> SandboxHandle: ) sandbox_id = f"sandbox-{sandbox_name}-{int(time.time() * 1000)}" + docker = parse_launch(config.sandbox_backend, config.settings) env_vars = { key: value for key, value in (config.user_env_vars or {}).items() @@ -382,6 +408,7 @@ async def _launch_sandbox(self, spec: _SandboxLaunchSpec) -> SandboxHandle: SANDBOX_TIMEOUT_ENV_VAR: str(config.timeout_seconds), "REPO_OWNER": config.repo_owner or "", "REPO_NAME": config.repo_name or "", + **docker_runtime_env(docker), } ) @@ -389,7 +416,7 @@ async def _launch_sandbox(self, spec: _SandboxLaunchSpec) -> SandboxHandle: include_github_cli_aliases = False snapshot_id: str | None = None if isinstance(spec.source, _BaseImageSource): - image = base_image + image = docker_base_image() if docker.enabled else base_image elif isinstance(spec.source, _RepositoryImageSource): image = modal.Image.from_id(spec.source.image_id) env_vars["FROM_REPO_IMAGE"] = "true" @@ -462,22 +489,30 @@ async def _launch_sandbox(self, spec: _SandboxLaunchSpec) -> SandboxHandle: "timeout": config.timeout_seconds, "workdir": "/workspace", "env": env_vars, - **_resource_kwargs(config.settings), + **launch_kwargs(docker), } if exposed_ports: create_kwargs["encrypted_ports"] = exposed_ports - try: - sandbox = await modal.Sandbox.create.aio( - "python", - "-m", - "sandbox_runtime.entrypoint", - **create_kwargs, + repository_image = isinstance(spec.source, _RepositoryImageSource) + if docker.enabled: + sandbox, adopted = await self._launch_docker_sandbox( + session_id=_session_identity(config.session_config), + sandbox_id=sandbox_id, + retire_sandbox_id=config.retire_sandbox_id, + create_kwargs=create_kwargs, + repository_image=repository_image, ) - except modal.exception.NotFoundError as e: - if isinstance(spec.source, _RepositoryImageSource): - raise RepositoryImageUnavailableError("repository image is unavailable") from e - raise + if adopted: + passwords = await self._read_access_passwords( + sandbox, + code_server_enabled=config.code_server_enabled, + vnc_enabled=config.vnc_enabled, + ) + code_server_password = passwords.get("CODE_SERVER_PASSWORD") + vnc_password = passwords.get(VNC_PASSWORD_ENV_VAR) + else: + sandbox = await _create_sandbox(create_kwargs, repository_image=repository_image) modal_object_id = sandbox.object_id ( code_server_url, @@ -509,6 +544,107 @@ async def _launch_sandbox(self, spec: _SandboxLaunchSpec) -> SandboxHandle: vnc_password=vnc_password, ttyd_url=ttyd_url, tunnel_urls=extra_tunnel_urls, + sandbox_backend=docker.backend, + ) + + async def _launch_docker_sandbox( + self, + *, + session_id: str, + sandbox_id: str, + retire_sandbox_id: str | None, + create_kwargs: dict[str, Any], + repository_image: bool, + ) -> tuple[modal.Sandbox, bool]: + """Create a Docker VM under a deterministic name, adopting an existing one. + + VM creation can outlive the control plane's HTTP request. One name per + session serializes generations at Modal even when a predecessor lookup + misses an in-flight create. Only matching generation tags permit adoption. + """ + if retire_sandbox_id: + await self._retire_docker_allocation(session_id, retire_sandbox_id) + name = docker_allocation_name(session_id) + tags = docker_allocation_tags(session_id, sandbox_id) + existing = await self._find_owned_docker_allocation(name, tags) + if existing is None: + try: + sandbox = await _create_sandbox( + {**create_kwargs, "name": name, "tags": tags}, + repository_image=repository_image, + ) + return sandbox, False + except modal.exception.AlreadyExistsError: + existing = await self._find_owned_docker_allocation(name, tags) + if existing is None: + raise + log.info( + "sandbox.docker_allocation_adopted", + sandbox_id=sandbox_id, + modal_object_id=existing.object_id, + ) + return existing, True + + @staticmethod + async def _read_access_passwords( + sandbox: modal.Sandbox, *, code_server_enabled: bool, vnc_enabled: bool + ) -> dict[str, str]: + """Recover only enabled service credentials from the owned VM's launch environment.""" + keys = [] + if code_server_enabled: + keys.append("CODE_SERVER_PASSWORD") + if vnc_enabled: + keys.append(VNC_PASSWORD_ENV_VAR) + if not keys: + return {} + process = await sandbox.exec.aio( + "python", + "-I", + "-c", + "import json, os, sys; print(json.dumps({k: os.environ.get(k) for k in sys.argv[1:]}))", + *keys, + timeout=ACCESS_PASSWORD_READ_TIMEOUT_SECONDS, + ) + output = await process.stdout.read.aio() + if await process.wait.aio() != 0: + raise RuntimeError("Could not recover adopted sandbox access credentials") + try: + passwords = json.loads(output) + except ValueError: + raise RuntimeError("Could not recover adopted sandbox access credentials") from None + if not isinstance(passwords, dict) or any( + not isinstance(passwords.get(key), str) or not passwords[key] for key in keys + ): + raise RuntimeError("Could not recover adopted sandbox access credentials") + return {key: passwords[key] for key in keys} + + @staticmethod + async def _find_owned_docker_allocation( + name: str, tags: dict[str, str] + ) -> modal.Sandbox | None: + try: + sandbox = await modal.Sandbox.from_name.aio(APP_NAME, name) + except modal.exception.NotFoundError: + return None + if await sandbox.get_tags.aio() != tags: + raise RuntimeError("Docker sandbox allocation ownership mismatch") + return sandbox + + async def _retire_docker_allocation(self, session_id: str, sandbox_id: str) -> None: + """Terminate a prior generation's named VM, only when its ownership tags match.""" + name = docker_allocation_name(session_id) + try: + sandbox = await modal.Sandbox.from_name.aio(APP_NAME, name) + except modal.exception.NotFoundError: + return + if await sandbox.get_tags.aio() != docker_allocation_tags(session_id, sandbox_id): + log.warn("sandbox.docker_allocation_retire_mismatch", sandbox_id=sandbox_id) + return + await sandbox.terminate.aio(wait=True) + log.info( + "sandbox.docker_allocation_retired", + sandbox_id=sandbox_id, + modal_object_id=sandbox.object_id, ) async def create_sandbox( @@ -586,6 +722,23 @@ async def take_snapshot( snapshot_timeout_seconds = min(int(timeout_seconds), SNAPSHOT_FILESYSTEM_TIMEOUT_SECONDS) if snapshot_timeout_seconds <= 0: raise TimeoutError("Insufficient time remains for a filesystem snapshot") + if handle.sandbox_backend == "modal-vm": + preparation_started = time.monotonic() + probe = await handle.modal_sandbox.exec.aio( + "python", + "-m", + "sandbox_runtime.docker_control", + "prepare", + timeout=min(snapshot_timeout_seconds, CONTROL_TIMEOUT_SECONDS), + ) + if await probe.wait.aio() != 0: + raise RuntimeError("Modal VM Docker shutdown preparation was not confirmed") + snapshot_timeout_seconds = min( + int(timeout_seconds - (time.monotonic() - preparation_started)), + SNAPSHOT_FILESYSTEM_TIMEOUT_SECONDS, + ) + if snapshot_timeout_seconds <= 0: + raise TimeoutError("Snapshot deadline expired during Docker preparation") image = await handle.modal_sandbox.snapshot_filesystem.aio(timeout=snapshot_timeout_seconds) # The image object_id is the unique identifier for this snapshot @@ -604,7 +757,11 @@ async def take_snapshot( return image_id async def stop_sandbox(self, sandbox_id: str) -> None: - """Terminate a provider sandbox by its immutable Modal object id.""" + """Resolve a pending reference if needed, then confirm immutable-ID retirement.""" + if sandbox_id.startswith("modal-vm-session:"): + handle = await self.get_sandbox_by_id(sandbox_id) + assert handle is not None and handle.modal_object_id is not None + sandbox_id = handle.modal_object_id try: sandbox = await modal.Sandbox.from_id.aio(sandbox_id) await sandbox.terminate.aio(wait=True) @@ -614,27 +771,49 @@ async def stop_sandbox(self, sandbox_id: str) -> None: async def get_sandbox_by_id(self, sandbox_id: str) -> SandboxHandle | None: """ - Get a sandbox handle by its ID. - - Uses Modal's Sandbox.from_id() to retrieve an existing sandbox. + Get a sandbox by immutable ID or a generation-checked pending reference. Args: - sandbox_id: The Modal sandbox ID + sandbox_id: The Modal sandbox ID or opaque VM session reference Returns: - SandboxHandle if found, None otherwise + SandboxHandle if found, None for a confirmed missing immutable ID. + Missing pending references remain ambiguous and raise an error. """ - try: - modal_sandbox = await modal.Sandbox.from_id.aio(sandbox_id) - return SandboxHandle( - sandbox_id=sandbox_id, - modal_sandbox=modal_sandbox, - status=SandboxStatus.READY, # Assume ready if we can retrieve it - created_at=time.time(), + if sandbox_id.startswith("modal-vm-session:"): + identity = json.loads(sandbox_id.removeprefix("modal-vm-session:")) + if ( + not isinstance(identity, list) + or len(identity) != 2 + or not all(isinstance(part, str) and part for part in identity) + ): + raise ValueError("Invalid pending VM reference") + session_id, generation_id = identity + modal_sandbox = await self._find_owned_docker_allocation( + docker_allocation_name(session_id), + docker_allocation_tags(session_id, generation_id), ) - except Exception as e: - log.warn("sandbox.lookup_error", sandbox_id=sandbox_id, exc=e) - return None + if modal_sandbox is None: + # An in-flight create can still materialize. Never report confirmed + # absence/retirement for an unresolved launch intent. + raise RuntimeError("VM launch identity is not yet visible") + else: + try: + modal_sandbox = await modal.Sandbox.from_id.aio(sandbox_id) + except modal.exception.NotFoundError: + return None + tags = await modal_sandbox.get_tags.aio() + backend = tags.get("openinspect_backend", "modal") + if backend not in ("modal", "modal-vm"): + raise ValueError("Unknown sandbox backend tag") + return SandboxHandle( + sandbox_backend="modal-vm" if backend == "modal-vm" else "modal", + sandbox_id=sandbox_id, + modal_object_id=modal_sandbox.object_id, + modal_sandbox=modal_sandbox, + status=SandboxStatus.READY, # Assume ready if we can retrieve it + created_at=time.time(), + ) async def restore_from_snapshot( self, @@ -650,6 +829,8 @@ async def restore_from_snapshot( vnc_enabled: bool = DEFAULT_VNC_ENABLED, agent_slack_notify_enabled: bool = False, settings: dict[str, Any] | None = None, + retire_sandbox_id: str | None = None, + sandbox_backend: ModalBackend = "modal", ) -> SandboxHandle: """ Create a new sandbox from a filesystem snapshot Image. @@ -700,7 +881,9 @@ async def restore_from_snapshot( code_server_enabled=code_server_enabled, vnc_enabled=vnc_enabled, agent_slack_notify_enabled=agent_slack_notify_enabled, + retire_sandbox_id=retire_sandbox_id, settings=settings, + sandbox_backend=sandbox_backend, ), source=_SnapshotImageSource( image_id=snapshot_image_id, diff --git a/packages/modal-infra/src/web_api.py b/packages/modal-infra/src/web_api.py index 835235ad87..2aaf09074c 100644 --- a/packages/modal-infra/src/web_api.py +++ b/packages/modal-infra/src/web_api.py @@ -12,6 +12,7 @@ """ import asyncio +import math import time from collections.abc import AsyncIterator from contextlib import asynccontextmanager @@ -36,6 +37,11 @@ ) from .clone_token import resolve_clone_token from .log_config import configure_logging, get_logger +from .sandbox.launch_policy import ( + DockerImageUnavailableError, + InvalidDockerSettingsError, + ModalBackend, +) configure_logging() log = get_logger("web_api") @@ -76,6 +82,8 @@ class CreateBuildSandboxRequest(_ModalRequestModel): user_env_vars: dict[str, str] | None = None build_execution_timeout_seconds: int | None = None provider_session_timeout_seconds: int | None = None + sandbox_settings: dict[str, Any] | None = None + sandbox_backend: ModalBackend = "modal" class StartBuildSandboxRequest(_ModalRequestModel): @@ -136,6 +144,8 @@ class CreateSandboxRequest(_RepositoryContextModel): vnc_enabled: bool | None = None agent_slack_notify_enabled: bool = False sandbox_settings: dict[str, Any] | None = None + sandbox_backend: ModalBackend = "modal" + retire_sandbox_id: str | None = None class RestoreSessionConfigRequest(_RepositoryContextModel): @@ -168,6 +178,8 @@ class RestoreSandboxRequest(_ModalRequestModel): vnc_enabled: bool | None = None agent_slack_notify_enabled: bool = False sandbox_settings: dict[str, Any] | None = None + sandbox_backend: ModalBackend = "modal" + retire_sandbox_id: str | None = None @dataclass @@ -208,6 +220,16 @@ async def _execute_endpoint( execution.http_status = e.status_code execution.outcome = "error" raise + except InvalidDockerSettingsError as e: + execution.http_status = 400 + execution.outcome = "error" + raise HTTPException(status_code=400, detail=str(e)) from e + except DockerImageUnavailableError as e: + # Not provisioned on this deployment: a permanent, actionable failure + # for the request, never a reason to launch the default sandbox. + execution.http_status = 501 + execution.outcome = "error" + raise HTTPException(status_code=501, detail="docker_not_available") from e except Exception as e: execution.http_status = 500 execution.outcome = "error" @@ -429,11 +451,13 @@ async def api_create_sandbox( ), agent_slack_notify_enabled=parsed_request.agent_slack_notify_enabled, settings=parsed_request.sandbox_settings or None, + sandbox_backend=parsed_request.sandbox_backend, timeout_seconds=( parsed_request.timeout_seconds if parsed_request.timeout_seconds is not None else DEFAULT_SANDBOX_TIMEOUT_SECONDS ), + retire_sandbox_id=parsed_request.retire_sandbox_id or None, ) try: @@ -464,10 +488,21 @@ async def api_create_sandbox( "vnc_password": handle.vnc_password, "ttyd_url": handle.ttyd_url, "tunnel_urls": handle.tunnel_urls, + "sandbox_backend": handle.sandbox_backend, }, } +@app.function(image=function_image) +def deployment_vm_image() -> str | None: + """Private SDK-only deployment handshake; preserve capability across selector cutover.""" + import os + + from .images.base import DOCKER_IMAGE_ID_ENV + + return os.environ.get(DOCKER_IMAGE_ID_ENV) + + @app.function(image=function_image) @fastapi_endpoint(method="GET") def api_health() -> dict: @@ -475,6 +510,25 @@ def api_health() -> dict: return {"success": True, "data": {"status": "healthy", "service": "open-inspect-modal"}} +def _snapshot_timeout_seconds(request: dict[str, Any]) -> float: + """Bound a capture by the caller's deadline without extending it.""" + from .sandbox.manager import SNAPSHOT_FILESYSTEM_TIMEOUT_SECONDS + + deadline_at_ms = request.get("deadline_at_ms") + if deadline_at_ms is None: + return SNAPSHOT_FILESYSTEM_TIMEOUT_SECONDS + if ( + isinstance(deadline_at_ms, bool) + or not isinstance(deadline_at_ms, (int, float)) + or not math.isfinite(deadline_at_ms) + ): + raise HTTPException(status_code=400, detail="deadline_at_ms must be a number") + timeout_seconds = (float(deadline_at_ms) / 1000) - time.time() + if timeout_seconds <= 0: + raise HTTPException(status_code=408, detail="snapshot deadline expired") + return timeout_seconds + + @app.function(image=function_image, secrets=[internal_api_secret]) @fastapi_endpoint(method="POST") async def api_snapshot_sandbox( @@ -524,28 +578,79 @@ async def api_snapshot_sandbox( from .sandbox.manager import SandboxManager manager = SandboxManager() + deadline_at_ms = request.get("deadline_at_ms") + timeout_seconds = _snapshot_timeout_seconds(request) + if request.get("sandbox_backend") == "modal-vm": + raise HTTPException(status_code=400, detail="Use the VM snapshot endpoint") + try: + async with asyncio.timeout(timeout_seconds): + handle = await manager.get_sandbox_by_id(sandbox_id) + if not handle: + raise HTTPException(status_code=404, detail=f"Sandbox not found: {sandbox_id}") + if handle.sandbox_backend == "modal-vm": + raise HTTPException(status_code=400, detail="Use the VM snapshot endpoint") + if deadline_at_ms is None: + image_id = await manager.take_snapshot(handle) + else: + image_id = await manager.take_snapshot(handle, timeout_seconds=timeout_seconds) + except (TimeoutError, ModalTimeoutError) as exc: + raise HTTPException(status_code=408, detail="snapshot deadline expired") from exc + return { + "success": True, + "data": { + "source_stopped": False, + "image_id": image_id, + "sandbox_id": sandbox_id, + }, + } - handle = await manager.get_sandbox_by_id(sandbox_id) - if not handle: - raise HTTPException(status_code=404, detail=f"Sandbox not found: {sandbox_id}") - deadline_at_ms = request.get("deadline_at_ms") - if deadline_at_ms is not None: - if isinstance(deadline_at_ms, bool) or not isinstance(deadline_at_ms, (int, float)): - raise HTTPException(status_code=400, detail="deadline_at_ms must be a number") - timeout_seconds = (deadline_at_ms / 1000) - time.time() - if timeout_seconds <= 0: - raise HTTPException(status_code=408, detail="snapshot deadline expired") - try: - image_id = await manager.take_snapshot(handle, timeout_seconds=timeout_seconds) - except (TimeoutError, ModalTimeoutError) as exc: - raise HTTPException(status_code=408, detail="snapshot deadline expired") from exc - else: - image_id = await manager.take_snapshot(handle) +@app.function(image=function_image, secrets=[internal_api_secret]) +@fastapi_endpoint(method="POST") +async def api_snapshot_vm_sandbox( + request: dict[str, Any], + authorization: str | None = Header(None), + x_trace_id: str | None = Header(None), + x_request_id: str | None = Header(None), + x_session_id: str | None = Header(None), + x_sandbox_id: str | None = Header(None), +) -> dict[str, Any]: + """Capture a prepared VM without retiring it; the control plane owns retirement.""" + async with _execute_endpoint( + endpoint_name="api_snapshot_vm_sandbox", + authorization=authorization, + trace_id=x_trace_id, + request_id=x_request_id, + session_id=x_session_id, + sandbox_id=x_sandbox_id, + ) as execution: + sandbox_id = request.get("sandbox_id") + execution.log_fields["sandbox_id"] = x_sandbox_id or sandbox_id + if not isinstance(sandbox_id, str) or not sandbox_id: + raise HTTPException(status_code=400, detail="sandbox_id is required") + if request.get("sandbox_backend") != "modal-vm": + raise HTTPException(status_code=400, detail="modal-vm backend confirmation is required") + from .sandbox.manager import SandboxManager + + manager = SandboxManager() + timeout_seconds = _snapshot_timeout_seconds(request) + try: + async with asyncio.timeout(timeout_seconds): + handle = await manager.get_sandbox_by_id(sandbox_id) + if handle is None or handle.sandbox_backend != "modal-vm": + raise HTTPException(status_code=400, detail="Terminal capture requires a VM") + source_id = handle.modal_object_id + if not source_id: + raise HTTPException(status_code=500, detail="VM source ID is unavailable") + image_id = await manager.take_snapshot(handle, timeout_seconds=timeout_seconds) + except (TimeoutError, ModalTimeoutError) as exc: + raise HTTPException(status_code=408, detail="snapshot deadline expired") from exc return { "success": True, "data": { + "source_stopped": False, + "source_id": source_id, "image_id": image_id, "sandbox_id": sandbox_id, }, @@ -712,6 +817,8 @@ async def api_restore_sandbox( ), agent_slack_notify_enabled=parsed_request.agent_slack_notify_enabled, settings=parsed_request.sandbox_settings or None, + sandbox_backend=parsed_request.sandbox_backend, + retire_sandbox_id=parsed_request.retire_sandbox_id or None, ) return { @@ -726,6 +833,7 @@ async def api_restore_sandbox( "vnc_password": handle.vnc_password, "ttyd_url": handle.ttyd_url, "tunnel_urls": handle.tunnel_urls, + "sandbox_backend": handle.sandbox_backend, }, } @@ -787,7 +895,7 @@ async def api_create_build_sandbox( status_code=400, detail="callback URLs must target the control plane" ) - provider_session_id = await ModalBuildSessionService().create( + launch = await ModalBuildSessionService().create( build_id=build_id, scope_kind=scope_kind, scope_id=scope_id, @@ -800,11 +908,16 @@ async def api_create_build_sandbox( user_env_vars=parsed_request.user_env_vars or None, build_execution_timeout_seconds=build_execution_timeout_seconds, timeout_seconds=provider_session_timeout_seconds, + sandbox_settings=parsed_request.sandbox_settings or None, + sandbox_backend=parsed_request.sandbox_backend, ) - execution.log_fields["sandbox_id"] = provider_session_id + execution.log_fields["sandbox_id"] = launch.provider_session_id return { "success": True, - "data": {"provider_session_id": provider_session_id}, + "data": { + "provider_session_id": launch.provider_session_id, + "sandbox_backend": launch.sandbox_backend, + }, } diff --git a/packages/modal-infra/tests/test_build_sandbox_lifecycle.py b/packages/modal-infra/tests/test_build_sandbox_lifecycle.py index 47c3937515..db011b58c5 100644 --- a/packages/modal-infra/tests/test_build_sandbox_lifecycle.py +++ b/packages/modal-infra/tests/test_build_sandbox_lifecycle.py @@ -8,7 +8,10 @@ import pytest -from sandbox_runtime.constants import IMAGE_BUILD_EXECUTION_TIMEOUT_ENV_VAR +from sandbox_runtime.constants import ( + DOCKER_ENABLED_ENV_VAR, + IMAGE_BUILD_EXECUTION_TIMEOUT_ENV_VAR, +) from sandbox_runtime.modal_image_build_start import MODAL_SANDBOX_ID_ENV from sandbox_runtime.repo_image_callback import ( BUILD_ID_ENV, @@ -28,6 +31,16 @@ from src.web_api import IMAGE_BUILD_FINALIZATION_GRACE_SECONDS +@pytest.fixture(autouse=True) +def no_remote_named_lookup(monkeypatch): + from modal.exception import NotFoundError + + monkeypatch.setattr("src.sandbox.build_session.modal.Sandbox.from_name", _async_method()) + from src.sandbox.build_session import modal + + modal.Sandbox.from_name.aio.side_effect = NotFoundError("not found") + + def _async_method(return_value=None): method = MagicMock() method.aio = AsyncMock(return_value=return_value) @@ -162,7 +175,7 @@ async def test_create_build_sandbox_runs_gated_entrypoint_and_scrubs_callback_en create = _async_method(sandbox) monkeypatch.setattr("src.sandbox.build_session.modal.Sandbox.create", create) - provider_session_id = await ModalBuildSessionService().create( + launch = await ModalBuildSessionService().create( build_id="build-1", scope_kind="repo", scope_id="acme/repo", @@ -185,7 +198,8 @@ async def test_create_build_sandbox_runs_gated_entrypoint_and_scrubs_callback_en timeout_seconds=1800, ) - assert provider_session_id == "modal-session-1" + assert launch.provider_session_id == "modal-session-1" + assert launch.sandbox_backend == "modal" args = create.aio.await_args.args kwargs = create.aio.await_args.kwargs assert args == ( @@ -195,6 +209,7 @@ async def test_create_build_sandbox_runs_gated_entrypoint_and_scrubs_callback_en "--await-modal-image-build-token-stdin-v1", ) assert kwargs["tags"] == { + "openinspect_backend": "modal", "openinspect_kind": "image-build", "openinspect_build_id": "build-1", "openinspect_scope_kind": "repo", @@ -370,3 +385,84 @@ async def test_terminate_build_sandbox_treats_provider_not_found_as_success(monk ) sandbox.terminate.aio.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("docker_enabled", [False, True]) +async def test_create_build_sandbox_selects_the_variant_from_frozen_settings( + monkeypatch, docker_enabled +): + sandbox = SimpleNamespace(object_id="modal-session-1") + create = _async_method(sandbox) + monkeypatch.setattr("src.sandbox.build_session.modal.Sandbox.create", create) + default_image = object() + docker_image = object() + monkeypatch.setattr("src.sandbox.build_session.base_image", default_image) + monkeypatch.setattr("src.images.base.docker_image", docker_image) + + launch = await ModalBuildSessionService().create( + build_id="build-1", + scope_kind="repo", + scope_id="acme/repo", + repositories=[{"repo_owner": "acme", "repo_name": "repo", "branch": "main"}], + callback_url="https://cp.test/image-builds/build-complete", + failure_callback_url="https://cp.test/image-builds/build-failed", + user_env_vars={DOCKER_ENABLED_ENV_VAR: "true"}, + sandbox_backend="modal-vm" if docker_enabled else "modal", + sandbox_settings=({"cpuCores": 2, "memoryMib": 4096} if docker_enabled else None), + ) + + assert launch.provider_session_id == "modal-session-1" + assert launch.sandbox_backend == ("modal-vm" if docker_enabled else "modal") + kwargs = create.aio.await_args.kwargs + assert kwargs["env"][DOCKER_ENABLED_ENV_VAR] == ("true" if docker_enabled else "false") + if docker_enabled: + assert kwargs["image"] is docker_image + assert kwargs["experimental_options"] == {"vm_runtime": True} + assert (kwargs["cpu"], kwargs["memory"]) == ((2, 2), 4096) + else: + assert kwargs["image"] is default_image + assert "experimental_options" not in kwargs + + +@pytest.mark.asyncio +async def test_build_create_retry_adopts_only_owned_backend_allocation(monkeypatch): + from src.sandbox.build_session import ModalBuildSessionService + + tags = { + "openinspect_kind": "image-build", + "openinspect_build_id": "build-1", + "openinspect_scope_kind": "repo", + "openinspect_scope_id": "acme/repo", + "openinspect_launch_protocol": "stdin-token-v1", + "openinspect_backend": "modal-vm", + } + sandbox = SimpleNamespace(object_id="sb-existing", get_tags=_async_method(tags)) + lookup = _async_method(sandbox) + create = _async_method() + monkeypatch.setattr("src.sandbox.build_session.modal.Sandbox.from_name", lookup) + monkeypatch.setattr("src.sandbox.build_session.modal.Sandbox.create", create) + service = ModalBuildSessionService() + launch = await service.create( + build_id="build-1", + sandbox_backend="modal-vm", + scope_kind="repo", + scope_id="acme/repo", + repositories=[{"repo_owner": "acme", "repo_name": "repo", "branch": "main"}], + callback_url="https://cp.test/complete", + failure_callback_url="https://cp.test/failed", + ) + assert launch.provider_session_id == "sb-existing" + create.aio.assert_not_awaited() + tags["openinspect_backend"] = "modal" + with pytest.raises(RuntimeError, match="ownership"): + await service.create( + build_id="build-1", + sandbox_backend="modal-vm", + scope_kind="repo", + scope_id="acme/repo", + repositories=[{"repo_owner": "acme", "repo_name": "repo", "branch": "main"}], + callback_url="https://cp.test/complete", + failure_callback_url="https://cp.test/failed", + ) + create.aio.assert_not_awaited() diff --git a/packages/modal-infra/tests/test_deploy.py b/packages/modal-infra/tests/test_deploy.py index 179c3b2eee..25be573e9d 100644 --- a/packages/modal-infra/tests/test_deploy.py +++ b/packages/modal-infra/tests/test_deploy.py @@ -8,9 +8,18 @@ from unittest.mock import Mock import deploy +import modal import pytest +@pytest.fixture(autouse=True) +def deployed_vm_reference(monkeypatch): + function = Mock() + function.remote.return_value = None + monkeypatch.setattr(deploy.modal.Function, "from_name", Mock(return_value=function)) + return function + + def test_deployment_rejects_missing_image_without_opt_in(monkeypatch, tmp_path) -> None: from src.images import base @@ -108,7 +117,11 @@ def test_eager_build_does_not_register_functions_before_image_exists() -> None: assert result.returncode == 0, result.stderr -def test_build_sandbox_image_eagerly_builds_against_deployed_app(monkeypatch, tmp_path) -> None: +@pytest.mark.parametrize("prior_vm_image", [None, "im-existing-vm"]) +def test_build_sandbox_image_eagerly_builds_against_deployed_app( + monkeypatch, tmp_path, deployed_vm_reference, prior_vm_image +) -> None: + deployed_vm_reference.remote.return_value = prior_vm_image deployed_app = object() lookup = Mock(return_value=deployed_app) build = Mock() @@ -136,10 +149,25 @@ def test_build_sandbox_image_eagerly_builds_against_deployed_app(monkeypatch, tm assert create.call_args.kwargs["env"] is plan["runtimeEnv"] assert create.call_args.kwargs["cpu"] == 2.0 sandbox.terminate.assert_called_once() - assert json.loads((tmp_path / "selected.json").read_text()) == { + expected = { "imageId": "im-verified", "buildHash": "packed-recipe", } + if prior_vm_image: + expected["dockerImageId"] = prior_vm_image + assert json.loads((tmp_path / "selected.json").read_text()) == expected + assert create.call_count == 1 # Reverse cutover retains capability without another VM build. + + +def test_reverse_cutover_fails_closed_if_existing_capability_cannot_be_read(deployed_vm_reference): + deployed_vm_reference.remote.side_effect = RuntimeError("lookup unavailable") + with pytest.raises(RuntimeError, match="lookup unavailable"): + deploy._deployed_vm_image() + + +def test_first_deployment_has_no_vm_capability(deployed_vm_reference): + deployed_vm_reference.remote.side_effect = modal.exception.NotFoundError("not deployed") + assert deploy._deployed_vm_image() is None def test_local_base_image_retains_its_packed_plan(monkeypatch, tmp_path) -> None: @@ -243,3 +271,124 @@ def test_src_modal_deploy_builds_sandbox_image_before_app_deploy(tmp_path: Path) "run python deploy.py --build-sandbox-image", "run modal deploy -m src", ] + + +def _verifying_sandbox(*, exit_codes: list[int]) -> Mock: + sandbox = Mock() + processes = [] + for code in exit_codes: + process = Mock(returncode=code) + process.stdout.read.return_value = "" + process.stderr.read.return_value = "boom" + processes.append(process) + sandbox.exec.side_effect = processes + return sandbox + + +@pytest.mark.parametrize("docker_verification_passes", [True, False]) +def test_docker_image_is_built_and_verified_on_the_vm_after_the_default_is_published( + monkeypatch, tmp_path, docker_verification_passes +) -> None: + deployed_app = object() + monkeypatch.setattr(deploy.modal.App, "lookup", Mock(return_value=deployed_app)) + monkeypatch.setattr(deploy, "base_image", Mock(build=Mock(), object_id="im-default")) + docker_build = Mock() + monkeypatch.setattr(deploy, "docker_image", Mock(build=docker_build, object_id="im-docker")) + plan = {"buildHash": "packed-recipe", "runtimeEnv": {"PACKED_PLAN": "true"}} + monkeypatch.setattr(deploy, "base_image_plan", plan) + default_sandbox = _verifying_sandbox(exit_codes=[0]) + docker_sandbox = _verifying_sandbox(exit_codes=[0, 0 if docker_verification_passes else 1]) + create = Mock(side_effect=[default_sandbox, docker_sandbox]) + monkeypatch.setattr(deploy.modal.Sandbox, "create", create) + record_path = tmp_path / "selected.json" + monkeypatch.setattr(deploy, "image_reference_path", lambda: record_path) + monkeypatch.setenv("OPENINSPECT_IMAGE_RESULT", str(tmp_path / "candidate.json")) + + if docker_verification_passes: + deploy.build_sandbox_image(with_docker=True) + else: + with pytest.raises(RuntimeError, match="verification failed"): + deploy.build_sandbox_image(with_docker=True) + + docker_build.assert_called_once_with(deployed_app) + assert "experimental_options" not in create.call_args_list[0].kwargs + vm_kwargs = create.call_args_list[1].kwargs + assert vm_kwargs["experimental_options"] == {"vm_runtime": True} + assert (vm_kwargs["cpu"], vm_kwargs["memory"]) == (2, 4096) + assert [call.args[1] for call in docker_sandbox.exec.call_args_list] == [ + "/app/verify/smoke_test.py", + "/app/verify/docker_smoke.py", + ][: len(docker_sandbox.exec.call_args_list)] + default_sandbox.terminate.assert_called_once() + docker_sandbox.terminate.assert_called_once() + record = json.loads(record_path.read_text()) + expected = {"imageId": "im-default", "buildHash": "packed-recipe"} + if docker_verification_passes: + expected["dockerImageId"] = "im-docker" + # A failed Docker variant never disturbs the verified default reference. + assert record == expected + + +def test_deployed_environment_carries_the_docker_image_when_provisioned( + monkeypatch, tmp_path +) -> None: + from src.images import base + + record_path = tmp_path / "built.json" + monkeypatch.setattr(base.modal, "is_local", lambda: True) + monkeypatch.setattr(base, "image_reference_path", lambda: record_path) + monkeypatch.setattr( + base, "local_image_plan", lambda: (tmp_path, {"buildHash": "current-recipe"}) + ) + monkeypatch.delenv("BUILD_MODAL_VM_IMAGE", raising=False) + + record_path.write_text(json.dumps({"buildHash": "current-recipe", "imageId": "im-built"})) + assert base.deployed_image_environment() == {base.IMAGE_ID_ENV: "im-built"} + + record_path.write_text( + json.dumps( + {"buildHash": "current-recipe", "imageId": "im-built", "dockerImageId": "im-docker"} + ) + ) + assert base.deployed_image_environment() == { + base.IMAGE_ID_ENV: "im-built", + base.DOCKER_IMAGE_ID_ENV: "im-docker", + } + + record_path.write_text(json.dumps({"buildHash": "current-recipe", "imageId": "im-built"})) + monkeypatch.setenv("BUILD_MODAL_VM_IMAGE", "true") + with pytest.raises(RuntimeError, match="Docker sandbox image"): + base.deployed_image_environment() + + +def test_docker_image_is_absent_in_a_deployment_that_never_provisioned_it(monkeypatch) -> None: + from src.images import base + + monkeypatch.setattr(base.modal, "is_local", lambda: False) + monkeypatch.delenv(base.DOCKER_IMAGE_ID_ENV, raising=False) + assert base._define_docker_image() is None + + monkeypatch.setenv(base.DOCKER_IMAGE_ID_ENV, "im-docker") + monkeypatch.setattr(base.modal.Image, "from_id", Mock(return_value="docker-image")) + assert base._define_docker_image() == "docker-image" + + +def test_local_docker_image_is_the_default_image_plus_the_docker_phase(monkeypatch) -> None: + from src.images import base + + image = Mock() + monkeypatch.setattr(base.modal, "is_local", lambda: True) + monkeypatch.setattr(base, "base_image", image) + + base._define_docker_image() + + image.run_commands.assert_called_once_with( + "bash /tmp/openinspect-image/packages/sandbox-images/install/install.sh docker" + ) + + +def test_docker_verification_resources_match_backend_defaults() -> None: + from src.sandbox.launch_policy import VM_DEFAULT_CPU_CORES, VM_DEFAULT_MEMORY_MIB + + assert deploy.DOCKER_VERIFICATION_CPU_CORES == VM_DEFAULT_CPU_CORES + assert deploy.DOCKER_VERIFICATION_MEMORY_MIB == VM_DEFAULT_MEMORY_MIB diff --git a/packages/modal-infra/tests/test_docker_launch.py b/packages/modal-infra/tests/test_docker_launch.py new file mode 100644 index 0000000000..cd53964542 --- /dev/null +++ b/packages/modal-infra/tests/test_docker_launch.py @@ -0,0 +1,107 @@ +"""Provider-local mapping from backend identity and resources to launch mechanics.""" + +import re + +import pytest + +from sandbox_runtime.constants import DOCKER_ENABLED_ENV_VAR +from src.images import base +from src.sandbox.launch_policy import ( + DockerImageUnavailableError, + InvalidDockerSettingsError, + ModalLaunch, + docker_allocation_name, + docker_allocation_tags, + docker_base_image, + docker_runtime_env, + launch_kwargs, + parse_launch, +) + + +@pytest.mark.parametrize("settings", [None, {}]) +def test_standard_defaults_preserve_existing_launch(settings): + launch = parse_launch("modal", settings) + + assert launch == ModalLaunch(backend="modal") + assert launch_kwargs(launch) == {} + assert docker_runtime_env(launch) == {DOCKER_ENABLED_ENV_VAR: "false"} + + +def test_launch_policy_maps_vm_backend_and_resources(): + launch = parse_launch("modal-vm", {"cpuCores": 2, "memoryMib": 4096}) + + assert launch == ModalLaunch(backend="modal-vm", cpu_cores=2.0, memory_mib=4096) + assert launch_kwargs(launch) == { + "experimental_options": {"vm_runtime": True}, + "cpu": (2.0, 2.0), + "memory": 4096, + } + assert docker_runtime_env(launch) == {DOCKER_ENABLED_ENV_VAR: "true"} + + +@pytest.mark.parametrize( + "settings", + [ + {"dockerEnabled": "true"}, + {"dockerEnabled": 1}, + {"dockerEnabled": None}, + {"dockerEnabled": True}, + {"cpuCores": -1}, + {"cpuCores": 0, "memoryMib": 4096}, + {"cpuCores": True, "memoryMib": 4096}, + {"cpuCores": 2, "memoryMib": "4096"}, + {"cpuCores": 2, "memoryMib": 4096.5}, + {"cpuCores": float("inf"), "memoryMib": 4096}, + ], +) +def test_malformed_resources_or_removed_settings_are_rejected(settings): + with pytest.raises(InvalidDockerSettingsError): + parse_launch("modal-vm", settings) + + +def test_docker_base_image_requires_provisioning(monkeypatch): + monkeypatch.setattr(base, "docker_image", None) + with pytest.raises(DockerImageUnavailableError): + docker_base_image() + + sentinel = object() + monkeypatch.setattr(base, "docker_image", sentinel) + assert docker_base_image() is sentinel + + +def test_allocation_name_is_stable_per_session_and_modal_safe(): + name = docker_allocation_name("session/with:odd chars") + assert name == docker_allocation_name("session/with:odd chars") + assert name != docker_allocation_name("other-session") + assert len(name) <= 64 + assert re.fullmatch(r"[a-zA-Z0-9-_.]+", name) + assert not re.fullmatch(r"ap-[a-zA-Z0-9]{22}", name) + + +def test_allocation_tags_bind_session_generation_and_backend(): + tags = docker_allocation_tags("session-1", "sandbox-1") + + assert tags["openinspect_kind"] == "session" + assert tags["openinspect_backend"] == "modal-vm" + assert tags != docker_allocation_tags("session-1", "sandbox-2") + for value in tags.values(): + assert re.fullmatch(r"[a-zA-Z0-9._-]{1,63}", value) + + +@pytest.mark.parametrize("settings", [None, {}, {"cpuCores": None, "memoryMib": None}]) +def test_vm_owns_defaults_for_absent_or_null_resources(settings): + assert launch_kwargs(parse_launch("modal-vm", settings)) == { + "cpu": (2, 2), + "memory": 4096, + "experimental_options": {"vm_runtime": True}, + } + + +@pytest.mark.parametrize( + "settings", [{"memoryMib": True}, {"memoryMib": 0}, {"cpuCores": float("nan")}] +) +def test_both_backends_validate_explicit_resources(settings): + for backend in ("modal", "modal-vm"): + with pytest.raises(InvalidDockerSettingsError): + parse_launch(backend, settings) diff --git a/packages/modal-infra/tests/test_sandbox_launch.py b/packages/modal-infra/tests/test_sandbox_launch.py index aea331a650..c0195b1508 100644 --- a/packages/modal-infra/tests/test_sandbox_launch.py +++ b/packages/modal-infra/tests/test_sandbox_launch.py @@ -1,5 +1,6 @@ """Behavior matrix for shared fresh, repository-image, and snapshot launches.""" +import asyncio import json from types import SimpleNamespace from unittest.mock import AsyncMock, Mock @@ -8,12 +9,19 @@ from sandbox_runtime.constants import ( CODE_SERVER_PORT_ENV_VAR, + DOCKER_ENABLED_ENV_VAR, EXPECTED_TUNNEL_PORTS_ENV_VAR, NOVNC_PORT_ENV_VAR, TTYD_PROXY_PORT_ENV_VAR, VNC_PASSWORD_ENV_VAR, ) from sandbox_runtime.types import SessionConfig +from src.sandbox.launch_policy import ( + DockerImageUnavailableError, + InvalidDockerSettingsError, + docker_allocation_name, + docker_allocation_tags, +) from src.sandbox.manager import ( RepositoryImageUnavailableError, SandboxConfig, @@ -136,6 +144,11 @@ async def test_launch_matrix_preserves_common_and_source_specific_behavior( assert kwargs["cpu"] == 1.5 assert kwargs["memory"] == 3072 assert kwargs["encrypted_ports"] == [9000, 9001, 9002, 3000] + # The default launch never touches the VM runtime or named allocations. + assert "experimental_options" not in kwargs + assert "name" not in kwargs + assert "tags" not in kwargs + assert env[DOCKER_ENABLED_ENV_VAR] == "false" assert env["CONTROL_PLANE_URL"] == "https://control.example" assert env["CUSTOM_ENV"] == "preserved" @@ -261,3 +274,359 @@ async def test_base_image_not_found_is_not_classified_as_repository_image(monkey await SandboxManager().create_sandbox(SandboxConfig(repo_owner="acme", repo_name="repo")) create.aio.assert_awaited_once() + + +DOCKER_SETTINGS = {"cpuCores": 2, "memoryMib": 4096} + + +def _docker_manager(monkeypatch) -> tuple[SandboxManager, dict, object]: + captured: dict = {} + docker_image = object() + monkeypatch.setattr("src.sandbox.manager.base_image", object()) + monkeypatch.setattr("src.images.base.docker_image", docker_image) + monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", _fake_create(captured)) + monkeypatch.setattr( + SandboxManager, + "_resolve_and_setup_tunnels", + AsyncMock(return_value=(None, None, None, {})), + ) + return SandboxManager(), captured, docker_image + + +def _docker_config(**overrides) -> SandboxConfig: + fields = { + "repo_owner": "acme", + "repo_name": "repo", + "sandbox_id": "sandbox-acme-repo-1700000000000", + "session_config": SessionConfig( + session_id="session-1", repo_owner="acme", repo_name="repo" + ), + "control_plane_url": "https://control.example", + "sandbox_auth_token": "token", + "user_env_vars": {DOCKER_ENABLED_ENV_VAR: "false", "CUSTOM_ENV": "preserved"}, + "settings": dict(DOCKER_SETTINGS), + "sandbox_backend": "modal-vm", + } + return SandboxConfig(**{**fields, **overrides}) + + +def _not_found(*_args, **_kwargs): + from modal.exception import NotFoundError + + raise NotFoundError("no sandbox") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("image_source", ["base", "repository", "snapshot"]) +async def test_docker_launch_selects_vm_runtime_and_named_allocation(monkeypatch, image_source): + manager, captured, docker_image = _docker_manager(monkeypatch) + artifact = object() + monkeypatch.setattr("src.sandbox.manager.modal.Image.from_id", lambda _id: artifact) + monkeypatch.setattr( + "src.sandbox.manager.modal.Sandbox.from_name", + SimpleNamespace(aio=AsyncMock(side_effect=_not_found)), + ) + + if image_source == "snapshot": + handle = await manager.restore_from_snapshot( + snapshot_image_id="snapshot-1", + session_config={"session_id": "session-1", "repo_owner": "acme", "repo_name": "repo"}, + sandbox_id="sandbox-acme-repo-1700000000000", + control_plane_url="https://control.example", + sandbox_auth_token="token", + user_env_vars={DOCKER_ENABLED_ENV_VAR: "false"}, + settings=dict(DOCKER_SETTINGS), + sandbox_backend="modal-vm", + ) + else: + handle = await manager.create_sandbox( + _docker_config(repo_image_id="repo-image-1" if image_source == "repository" else None) + ) + + kwargs = captured["kwargs"] + assert kwargs["image"] is (docker_image if image_source == "base" else artifact) + assert kwargs["experimental_options"] == {"vm_runtime": True} + assert kwargs["cpu"] == (2.0, 2.0) + assert kwargs["memory"] == 4096 + assert kwargs["name"] == docker_allocation_name("session-1") + assert kwargs["tags"] == docker_allocation_tags("session-1", "sandbox-acme-repo-1700000000000") + # The trusted signal wins over any user-supplied value. + assert kwargs["env"][DOCKER_ENABLED_ENV_VAR] == "true" + assert handle.sandbox_backend == "modal-vm" + + +@pytest.mark.asyncio +async def test_docker_launch_without_a_provisioned_image_never_uses_the_default(monkeypatch): + manager, captured, _ = _docker_manager(monkeypatch) + monkeypatch.setattr("src.images.base.docker_image", None) + + with pytest.raises(DockerImageUnavailableError): + await manager.create_sandbox(_docker_config()) + + assert "kwargs" not in captured + + +@pytest.mark.asyncio +async def test_malformed_docker_setting_fails_before_any_launch(monkeypatch): + manager, captured, _ = _docker_manager(monkeypatch) + + with pytest.raises(InvalidDockerSettingsError): + await manager.create_sandbox(_docker_config(settings={"dockerEnabled": "true"})) + + assert "kwargs" not in captured + + +@pytest.mark.asyncio +async def test_docker_launch_adopts_an_existing_owned_allocation(monkeypatch): + manager, captured, _ = _docker_manager(monkeypatch) + tags = docker_allocation_tags("session-1", "sandbox-acme-repo-1700000000000") + existing = SimpleNamespace(object_id="modal-existing", get_tags=AsyncMock(return_value=tags)) + existing.get_tags.aio = existing.get_tags + from_name = AsyncMock(return_value=existing) + monkeypatch.setattr( + "src.sandbox.manager.modal.Sandbox.from_name", SimpleNamespace(aio=from_name) + ) + + handle = await manager.create_sandbox(_docker_config()) + + assert "kwargs" not in captured + assert handle.modal_object_id == "modal-existing" + from_name.assert_awaited_once_with("open-inspect", docker_allocation_name("session-1")) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("create_race", [False, True]) +@pytest.mark.parametrize("image_source", ["base", "snapshot"]) +async def test_docker_retry_returns_the_original_access_credentials( + monkeypatch, create_race, image_source +): + from modal.exception import AlreadyExistsError, NotFoundError + + manager, captured, _ = _docker_manager(monkeypatch) + monkeypatch.setattr("src.sandbox.manager.modal.Image.from_id", lambda _id: object()) + monkeypatch.setattr( + SandboxManager, "_generate_code_server_password", Mock(side_effect=["original", "new"]) + ) + monkeypatch.setattr( + SandboxManager, "_generate_vnc_password", Mock(side_effect=["old-vnc", "new-vnc"]) + ) + from_name = AsyncMock(side_effect=NotFoundError("not created")) + monkeypatch.setattr( + "src.sandbox.manager.modal.Sandbox.from_name", SimpleNamespace(aio=from_name) + ) + + async def launch(): + config = _docker_config(code_server_enabled=True, vnc_enabled=True) + if image_source == "base": + return await manager.create_sandbox(config) + return await manager.restore_from_snapshot( + snapshot_image_id="snapshot-1", + session_config=config.session_config, + sandbox_id=config.sandbox_id, + code_server_enabled=True, + vnc_enabled=True, + settings=config.settings, + sandbox_backend=config.sandbox_backend, + ) + + original = await launch() + original_env = captured["kwargs"]["env"] + credential_output = json.dumps( + {key: original_env[key] for key in ("CODE_SERVER_PASSWORD", VNC_PASSWORD_ENV_VAR)} + ) + process = SimpleNamespace( + stdout=SimpleNamespace(read=SimpleNamespace(aio=AsyncMock(return_value=credential_output))), + wait=SimpleNamespace(aio=AsyncMock(return_value=0)), + ) + existing = SimpleNamespace( + object_id=original.modal_object_id, + get_tags=SimpleNamespace(aio=AsyncMock(return_value=captured["kwargs"]["tags"])), + exec=SimpleNamespace(aio=AsyncMock(return_value=process)), + ) + from_name.side_effect = [NotFoundError("racing"), existing] if create_race else [existing] + create = AsyncMock(side_effect=AlreadyExistsError("already created")) + monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", SimpleNamespace(aio=create)) + + adopted = await launch() + + assert adopted.modal_object_id == original.modal_object_id + assert adopted.code_server_password == original.code_server_password == "original" + assert adopted.vnc_password == original.vnc_password == "old-vnc" + assert create.await_count == int(create_race) + assert existing.exec.aio.call_args.args[-2:] == ("CODE_SERVER_PASSWORD", VNC_PASSWORD_ENV_VAR) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "output,exit_code", [("{}", 0), ('{"CODE_SERVER_PASSWORD": ""}', 0), ("invalid", 0), ("", 1)] +) +async def test_docker_adoption_fails_if_original_credentials_cannot_be_recovered( + monkeypatch, output, exit_code +): + manager, captured, _ = _docker_manager(monkeypatch) + process = SimpleNamespace( + stdout=SimpleNamespace(read=SimpleNamespace(aio=AsyncMock(return_value=output))), + wait=SimpleNamespace(aio=AsyncMock(return_value=exit_code)), + ) + existing = SimpleNamespace( + object_id="modal-existing", + get_tags=SimpleNamespace( + aio=AsyncMock( + return_value=docker_allocation_tags("session-1", "sandbox-acme-repo-1700000000000") + ) + ), + exec=SimpleNamespace(aio=AsyncMock(return_value=process)), + ) + monkeypatch.setattr( + "src.sandbox.manager.modal.Sandbox.from_name", + SimpleNamespace(aio=AsyncMock(return_value=existing)), + ) + + with pytest.raises(RuntimeError, match="Could not recover adopted sandbox access credentials"): + await manager.create_sandbox(_docker_config(code_server_enabled=True)) + + assert "kwargs" not in captured + manager._resolve_and_setup_tunnels.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_docker_launch_refuses_a_same_named_allocation_it_does_not_own(monkeypatch): + manager, captured, _ = _docker_manager(monkeypatch) + foreign = SimpleNamespace( + object_id="modal-foreign", + get_tags=AsyncMock(return_value={"openinspect_kind": "session"}), + ) + foreign.get_tags.aio = foreign.get_tags + monkeypatch.setattr( + "src.sandbox.manager.modal.Sandbox.from_name", + SimpleNamespace(aio=AsyncMock(return_value=foreign)), + ) + + with pytest.raises(RuntimeError, match="ownership mismatch"): + await manager.create_sandbox(_docker_config()) + + assert "kwargs" not in captured + + +@pytest.mark.asyncio +async def test_docker_launch_retires_the_prior_generation_only_when_owned(monkeypatch): + manager, captured, _ = _docker_manager(monkeypatch) + prior_tags = docker_allocation_tags("session-1", "sandbox-acme-repo-1699999999999") + prior = SimpleNamespace( + object_id="modal-prior", + get_tags=AsyncMock(return_value=prior_tags), + terminate=AsyncMock(), + ) + prior.get_tags.aio = prior.get_tags + prior.terminate.aio = prior.terminate + prior_name = docker_allocation_name("session-1") + + async def from_name(_app, name): + if name == prior_name and not prior.terminate.await_count: + return prior + _not_found() + + monkeypatch.setattr( + "src.sandbox.manager.modal.Sandbox.from_name", SimpleNamespace(aio=from_name) + ) + + await manager.create_sandbox( + _docker_config(retire_sandbox_id="sandbox-acme-repo-1699999999999") + ) + + prior.terminate.assert_awaited_once_with(wait=True) + assert captured["kwargs"]["name"] == docker_allocation_name("session-1") + + # A prior allocation with foreign tags is left alone. + prior.terminate.reset_mock() + prior.get_tags = AsyncMock(return_value={"openinspect_kind": "other"}) + prior.get_tags.aio = prior.get_tags + with pytest.raises(RuntimeError, match="ownership mismatch"): + await manager.create_sandbox( + _docker_config(retire_sandbox_id="sandbox-acme-repo-1699999999999") + ) + prior.terminate.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_late_predecessor_cannot_materialize_beside_successor(monkeypatch): + from modal.exception import AlreadyExistsError, NotFoundError + + manager, _, _ = _docker_manager(monkeypatch) + predecessor_name = docker_allocation_name("session-1") + predecessor = SimpleNamespace( + object_id="late-predecessor", + get_tags=SimpleNamespace( + aio=AsyncMock(return_value=docker_allocation_tags("session-1", "prior")) + ), + ) + lookup = AsyncMock( + side_effect=[NotFoundError("still creating"), NotFoundError("still creating"), predecessor] + ) + monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.from_name", SimpleNamespace(aio=lookup)) + + async def create(kwargs, *, repository_image): + # Provider-side naming wins the race after both client lookups missed it. + if kwargs["name"] == predecessor_name: + raise AlreadyExistsError("predecessor won the name") + return SimpleNamespace(object_id="duplicate-successor") + + monkeypatch.setattr("src.sandbox.manager._create_sandbox", create) + with pytest.raises(RuntimeError, match="ownership mismatch"): + await manager._launch_docker_sandbox( + session_id="session-1", + sandbox_id="successor", + retire_sandbox_id="prior", + create_kwargs={}, + repository_image=False, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("termination_fails", [False, True]) +async def test_docker_successor_waits_for_confirmed_predecessor_retirement( + monkeypatch, termination_fails +): + manager, captured, _ = _docker_manager(monkeypatch) + termination_requested = asyncio.Event() + termination_finished = asyncio.Event() + + async def terminate(*, wait=False): + termination_requested.set() + if wait: + await termination_finished.wait() + if termination_fails: + raise RuntimeError("termination unconfirmed") + + prior = SimpleNamespace( + object_id="modal-prior", + get_tags=SimpleNamespace( + aio=AsyncMock(return_value=docker_allocation_tags("session-1", "sandbox-prior")) + ), + terminate=SimpleNamespace(aio=terminate), + ) + from modal.exception import NotFoundError + + monkeypatch.setattr( + "src.sandbox.manager.modal.Sandbox.from_name", + SimpleNamespace(aio=AsyncMock(side_effect=[prior, NotFoundError("no successor")])), + ) + launch = asyncio.create_task( + manager.create_sandbox(_docker_config(retire_sandbox_id="sandbox-prior")) + ) + try: + await asyncio.wait_for(termination_requested.wait(), timeout=1) + assert not launch.done() + assert "kwargs" not in captured + + termination_finished.set() + if termination_fails: + with pytest.raises(RuntimeError, match="termination unconfirmed"): + await launch + assert "kwargs" not in captured + else: + await launch + assert "kwargs" in captured + finally: + launch.cancel() + await asyncio.gather(launch, return_exceptions=True) diff --git a/packages/modal-infra/tests/test_sandbox_resources.py b/packages/modal-infra/tests/test_sandbox_resources.py index cc2aa8b105..cb8d7a1b90 100644 --- a/packages/modal-infra/tests/test_sandbox_resources.py +++ b/packages/modal-infra/tests/test_sandbox_resources.py @@ -4,26 +4,27 @@ import pytest -from src.sandbox.manager import SandboxConfig, SandboxManager, _resource_kwargs +from src.sandbox.launch_policy import launch_kwargs, parse_launch +from src.sandbox.manager import SandboxConfig, SandboxManager class TestResourceKwargs: - """_resource_kwargs maps sandbox settings to Modal create kwargs.""" + """Provider-local launch policy maps resource settings to Modal kwargs.""" def test_empty_settings(self): - assert _resource_kwargs({}) == {} + assert launch_kwargs(parse_launch("modal", {})) == {} def test_maps_cpu_and_memory(self): - assert _resource_kwargs({"cpuCores": 2, "memoryMib": 4096}) == { + assert launch_kwargs(parse_launch("modal", {"cpuCores": 2, "memoryMib": 4096})) == { "cpu": 2.0, "memory": 4096, } def test_allows_fractional_cpu(self): - assert _resource_kwargs({"cpuCores": 0.5}) == {"cpu": 0.5} + assert launch_kwargs(parse_launch("modal", {"cpuCores": 0.5})) == {"cpu": 0.5} def test_independent_fields(self): - assert _resource_kwargs({"memoryMib": 2048}) == {"memory": 2048} + assert launch_kwargs(parse_launch("modal", {"memoryMib": 2048})) == {"memory": 2048} def _fake_create(captured: dict): diff --git a/packages/modal-infra/tests/test_snapshot_timeout.py b/packages/modal-infra/tests/test_snapshot_timeout.py index ef931d3167..a4e4eb8ff6 100644 --- a/packages/modal-infra/tests/test_snapshot_timeout.py +++ b/packages/modal-infra/tests/test_snapshot_timeout.py @@ -7,7 +7,9 @@ import pytest from modal.exception import NotFoundError as ModalNotFoundError +from sandbox_runtime.docker_control import CONTROL_TIMEOUT_SECONDS from sandbox_runtime.types import SandboxStatus +from src.sandbox.launch_policy import docker_allocation_tags from src.sandbox.manager import ( SNAPSHOT_FILESYSTEM_TIMEOUT_SECONDS, SandboxHandle, @@ -15,6 +17,59 @@ ) +@pytest.mark.asyncio +async def test_pending_vm_reference_recovers_owned_allocation(monkeypatch): + sandbox = SimpleNamespace( + object_id="sb-owned", + get_tags=_async_method(docker_allocation_tags("session", "generation")), + ) + lookup = _async_method(sandbox) + monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.from_name", lookup) + handle = await SandboxManager().get_sandbox_by_id('modal-vm-session:["session","generation"]') + assert handle is not None and handle.modal_sandbox is sandbox + assert handle.sandbox_backend == "modal-vm" + + +@pytest.mark.asyncio +async def test_pending_vm_reference_stops_only_its_owned_allocation(monkeypatch): + sandbox = SimpleNamespace( + object_id="sb-owned", + get_tags=_async_method(docker_allocation_tags("session", "generation")), + terminate=_async_method(), + ) + from_name = _async_method(sandbox) + from_id = _async_method(sandbox) + monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.from_name", from_name) + monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.from_id", from_id) + + await SandboxManager().stop_sandbox('modal-vm-session:["session","generation"]') + + from_id.aio.assert_awaited_once_with("sb-owned") + sandbox.terminate.aio.assert_awaited_once_with(wait=True) + + +@pytest.mark.asyncio +async def test_pending_vm_reference_never_confirms_absence(monkeypatch): + lookup = _async_method() + lookup.aio.side_effect = ModalNotFoundError("not visible yet") + monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.from_name", lookup) + with pytest.raises(RuntimeError, match="not yet visible"): + await SandboxManager().stop_sandbox('modal-vm-session:["session","generation"]') + + +@pytest.mark.asyncio +async def test_pending_vm_reference_cannot_stop_another_generation(monkeypatch): + sandbox = SimpleNamespace( + object_id="sb-owned", + get_tags=_async_method(docker_allocation_tags("session", "other")), + terminate=_async_method(), + ) + monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.from_name", _async_method(sandbox)) + with pytest.raises(RuntimeError, match="ownership mismatch"): + await SandboxManager().stop_sandbox('modal-vm-session:["session","generation"]') + sandbox.terminate.aio.assert_not_awaited() + + def _async_method(return_value=None): method = MagicMock() method.aio = AsyncMock(return_value=return_value) @@ -42,7 +97,7 @@ async def test_take_snapshot_passes_explicit_timeout(): @pytest.mark.asyncio async def test_get_sandbox_by_id_awaits_async_lookup(monkeypatch): - modal_sandbox = SimpleNamespace() + modal_sandbox = SimpleNamespace(object_id="sb-owned", get_tags=_async_method({})) from_id = _async_method(modal_sandbox) monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.from_id", from_id) @@ -50,10 +105,37 @@ async def test_get_sandbox_by_id_awaits_async_lookup(monkeypatch): assert handle is not None assert handle.modal_sandbox is modal_sandbox + assert handle.modal_object_id == "sb-owned" from_id.assert_not_called() from_id.aio.assert_awaited_once_with("sandbox-1") +@pytest.mark.asyncio +async def test_tag_lookup_failure_is_not_sandbox_absence(monkeypatch): + tags = _async_method() + tags.aio.side_effect = RuntimeError("tag service unavailable") + monkeypatch.setattr( + "src.sandbox.manager.modal.Sandbox.from_id", + _async_method(SimpleNamespace(object_id="sb-owned", get_tags=tags)), + ) + with pytest.raises(RuntimeError, match="tag service unavailable"): + await SandboxManager().get_sandbox_by_id("sandbox-1") + + +@pytest.mark.asyncio +async def test_unknown_backend_tag_is_explicitly_rejected(monkeypatch): + monkeypatch.setattr( + "src.sandbox.manager.modal.Sandbox.from_id", + _async_method( + SimpleNamespace( + object_id="sb-owned", get_tags=_async_method({"openinspect_backend": "unknown"}) + ) + ), + ) + with pytest.raises(ValueError, match="Unknown sandbox backend tag"): + await SandboxManager().get_sandbox_by_id("sandbox-1") + + @pytest.mark.asyncio @pytest.mark.parametrize("budget, expected", [(179.8, 179), (500, 300)]) async def test_take_snapshot_bounds_whole_second_timeout(budget, expected): @@ -142,3 +224,63 @@ async def test_stop_sandbox_propagates_explicit_cancellation(monkeypatch): with pytest.raises(asyncio.CancelledError): await SandboxManager().stop_sandbox("sandbox-1") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("exit_code", [0, 1]) +async def test_vm_capture_requires_docker_preparation(exit_code): + process = SimpleNamespace(wait=_async_method(exit_code)) + execute = _async_method(process) + snapshot = _async_method(SimpleNamespace(object_id="im-vm")) + handle = SandboxHandle( + sandbox_id="sb-vm", + sandbox_backend="modal-vm", + status=SandboxStatus.READY, + created_at=0, + modal_sandbox=SimpleNamespace(exec=execute, snapshot_filesystem=snapshot), + ) + if exit_code: + with pytest.raises(RuntimeError, match="preparation"): + await SandboxManager().take_snapshot(handle) + snapshot.aio.assert_not_awaited() + else: + assert await SandboxManager().take_snapshot(handle) == "im-vm" + snapshot.aio.assert_awaited_once() + assert execute.aio.call_args.args == ( + "python", + "-m", + "sandbox_runtime.docker_control", + "prepare", + ) + assert execute.aio.call_args.kwargs["timeout"] == CONTROL_TIMEOUT_SECONDS + + +@pytest.mark.asyncio +@pytest.mark.parametrize("elapsed", [9, 10]) +async def test_vm_preparation_consumes_capture_budget(monkeypatch, elapsed): + clock = SimpleNamespace(time=lambda: 1000, monotonic=lambda: 0) + monkeypatch.setattr("src.sandbox.manager.time", clock) + + async def prepare(): + clock.monotonic = lambda: elapsed + return 0 + + wait = _async_method() + wait.aio.side_effect = prepare + snapshot = _async_method(SimpleNamespace(object_id="im-vm")) + handle = SandboxHandle( + sandbox_id="sb-vm", + sandbox_backend="modal-vm", + status=SandboxStatus.READY, + created_at=0, + modal_sandbox=SimpleNamespace( + exec=_async_method(SimpleNamespace(wait=wait)), snapshot_filesystem=snapshot + ), + ) + if elapsed == 10: + with pytest.raises(TimeoutError): + await SandboxManager().take_snapshot(handle, timeout_seconds=10) + snapshot.aio.assert_not_awaited() + else: + await SandboxManager().take_snapshot(handle, timeout_seconds=10) + snapshot.aio.assert_awaited_once_with(timeout=1) diff --git a/packages/modal-infra/tests/test_web_api_build_sandbox.py b/packages/modal-infra/tests/test_web_api_build_sandbox.py index 4ca2d645d0..9c9f27f3ca 100644 --- a/packages/modal-infra/tests/test_web_api_build_sandbox.py +++ b/packages/modal-infra/tests/test_web_api_build_sandbox.py @@ -9,7 +9,11 @@ from sandbox_runtime.types import SandboxStatus from src import web_api -from src.sandbox.build_session import DEFAULT_BUILD_TIMEOUT_SECONDS, MAX_BUILD_TIMEOUT_SECONDS +from src.sandbox.build_session import ( + DEFAULT_BUILD_TIMEOUT_SECONDS, + MAX_BUILD_TIMEOUT_SECONDS, + BuildSessionLaunch, +) from src.sandbox.manager import SandboxHandle, SandboxManager REPOSITORIES = [{"repo_owner": "acme", "repo_name": "repo", "branch": "main"}] @@ -19,11 +23,90 @@ } +@pytest.mark.asyncio +async def test_vm_capture_leaves_source_alive_until_control_plane_commits(monkeypatch): + monkeypatch.setattr(web_api, "require_auth", lambda _authorization: None) + handle = SimpleNamespace(sandbox_backend="modal-vm", modal_object_id="sb-immutable") + manager = SimpleNamespace( + get_sandbox_by_id=AsyncMock(return_value=handle), + take_snapshot=AsyncMock(side_effect=["im-first", "im-retry"]), + stop_sandbox=AsyncMock(), + ) + monkeypatch.setattr("src.sandbox.manager.SandboxManager", lambda: manager) + request = {"sandbox_id": "sb-vm", "sandbox_backend": "modal-vm"} + + first = await _call_vm_snapshot(request) + second = await _call_vm_snapshot(request) + + assert first["data"] == { + "source_stopped": False, + "source_id": "sb-immutable", + "image_id": "im-first", + "sandbox_id": "sb-vm", + } + assert second["data"]["image_id"] == "im-retry" + assert manager.take_snapshot.await_count == 2 + manager.stop_sandbox.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_vm_capture_rejects_non_vm_source(monkeypatch): + monkeypatch.setattr(web_api, "require_auth", lambda _authorization: None) + manager = SimpleNamespace( + get_sandbox_by_id=AsyncMock(return_value=SimpleNamespace(sandbox_backend="modal")), + take_snapshot=AsyncMock(), + ) + monkeypatch.setattr("src.sandbox.manager.SandboxManager", lambda: manager) + with pytest.raises(web_api.HTTPException, match="Terminal capture requires a VM"): + await _call_vm_snapshot({"sandbox_id": "sb-standard", "sandbox_backend": "modal-vm"}) + manager.take_snapshot.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_vm_capture_failure_never_retires(monkeypatch): + monkeypatch.setattr(web_api, "require_auth", lambda _authorization: None) + manager = SimpleNamespace( + get_sandbox_by_id=AsyncMock( + return_value=SimpleNamespace(sandbox_backend="modal-vm", modal_object_id="sb-immutable") + ), + take_snapshot=AsyncMock(side_effect=RuntimeError("capture failed")), + stop_sandbox=AsyncMock(), + ) + monkeypatch.setattr("src.sandbox.manager.SandboxManager", lambda: manager) + with pytest.raises(web_api.HTTPException): + await _call_vm_snapshot({"sandbox_id": "sb-vm", "sandbox_backend": "modal-vm"}) + manager.stop_sandbox.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_generic_snapshot_rejects_vm_without_capture_or_retirement(monkeypatch): + monkeypatch.setattr(web_api, "require_auth", lambda _authorization: None) + manager = SimpleNamespace(get_sandbox_by_id=AsyncMock(), take_snapshot=AsyncMock()) + monkeypatch.setattr("src.sandbox.manager.SandboxManager", lambda: manager) + with pytest.raises(web_api.HTTPException, match="Use the VM snapshot endpoint"): + await _call_generic_snapshot({"sandbox_id": "sb-vm", "sandbox_backend": "modal-vm"}) + manager.get_sandbox_by_id.assert_not_awaited() + manager.take_snapshot.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_generic_snapshot_rejects_unlabeled_vm_source(monkeypatch): + monkeypatch.setattr(web_api, "require_auth", lambda _authorization: None) + manager = SimpleNamespace( + get_sandbox_by_id=AsyncMock(return_value=SimpleNamespace(sandbox_backend="modal-vm")), + take_snapshot=AsyncMock(), + ) + monkeypatch.setattr("src.sandbox.manager.SandboxManager", lambda: manager) + with pytest.raises(web_api.HTTPException, match="Use the VM snapshot endpoint"): + await _call_generic_snapshot({"sandbox_id": "sb-vm"}) + manager.take_snapshot.assert_not_awaited() + + def _patch_dependencies(monkeypatch: pytest.MonkeyPatch): monkeypatch.setattr(web_api, "require_auth", lambda _authorization: None) monkeypatch.setattr(web_api, "validate_control_plane_url", lambda _url: True) service = SimpleNamespace( - create=AsyncMock(return_value="modal-session-1"), + create=AsyncMock(return_value=BuildSessionLaunch("modal-session-1", "modal")), start=AsyncMock(), terminate=AsyncMock(), snapshot=AsyncMock(return_value="modal-image-1"), @@ -55,6 +138,17 @@ async def _call_generic_snapshot(request: dict) -> dict: ) +async def _call_vm_snapshot(request: dict) -> dict: + return await web_api.api_snapshot_vm_sandbox.get_raw_f()( + request, + authorization="Bearer test", + x_trace_id=None, + x_request_id=None, + x_session_id=None, + x_sandbox_id=None, + ) + + async def _call_generic_stop(request: dict) -> dict: return await web_api.api_stop_sandbox.get_raw_f()( request, @@ -104,7 +198,7 @@ async def test_create_build_sandbox_forwards_callback_context_and_returns_provid assert result == { "success": True, - "data": {"provider_session_id": "modal-session-1"}, + "data": {"provider_session_id": "modal-session-1", "sandbox_backend": "modal"}, } service.create.assert_awaited_once_with( build_id="imgb-1", @@ -119,6 +213,8 @@ async def test_create_build_sandbox_forwards_callback_context_and_returns_provid user_env_vars={"FOO": "bar"}, build_execution_timeout_seconds=DEFAULT_BUILD_TIMEOUT_SECONDS, timeout_seconds=2400, + sandbox_settings=None, + sandbox_backend="modal", ) @@ -145,6 +241,36 @@ async def test_create_build_sandbox_adds_finalization_grace_to_default_timeout(m ) +@pytest.mark.asyncio +@pytest.mark.parametrize("docker_enabled", [False, True]) +async def test_create_build_sandbox_returns_the_provider_confirmed_variant( + monkeypatch, docker_enabled +): + service = _patch_dependencies(monkeypatch) + service.create.return_value = BuildSessionLaunch( + "modal-session-1", "modal-vm" if docker_enabled else "modal" + ) + settings = {"cpuCores": 2, "memoryMib": 4096} + + result = await _call( + web_api.api_create_build_sandbox, + { + "scope_kind": "repo", + "scope_id": "acme/repo", + "build_id": "imgb-1", + "repositories": REPOSITORIES, + "sandbox_settings": settings, + **CALLBACK_CONTEXT, + }, + ) + + assert result["data"] == { + "provider_session_id": "modal-session-1", + "sandbox_backend": "modal-vm" if docker_enabled else "modal", + } + assert service.create.await_args.kwargs["sandbox_settings"] == settings + + @pytest.mark.asyncio @pytest.mark.parametrize( ("payload_callbacks", "missing_field"), @@ -597,7 +723,7 @@ def reject_auth(_authorization): @pytest.mark.asyncio async def test_generic_snapshot_reason_cannot_select_build_identity_rules(monkeypatch): monkeypatch.setattr(web_api, "require_auth", lambda _authorization: None) - handle = SimpleNamespace() + handle = SimpleNamespace(sandbox_backend="modal") manager = SimpleNamespace( get_sandbox_by_id=AsyncMock(return_value=handle), take_snapshot=AsyncMock(return_value="im-session-1"), @@ -669,7 +795,7 @@ async def test_generic_snapshot_rejects_expired_deadline_without_provider_call(m assert exc.value.status_code == 408 assert exc.value.detail == "snapshot deadline expired" - get_sandbox_by_id.assert_awaited_once_with("modal-session-1") + get_sandbox_by_id.assert_not_awaited() snapshot_filesystem.aio.assert_not_awaited() @@ -687,7 +813,7 @@ async def test_generic_snapshot_rejects_boolean_deadline_without_provider_call( assert exc.value.status_code == 400 assert exc.value.detail == "deadline_at_ms must be a number" - get_sandbox_by_id.assert_awaited_once_with("modal-session-1") + get_sandbox_by_id.assert_not_awaited() snapshot_filesystem.aio.assert_not_awaited() diff --git a/packages/modal-infra/tests/test_web_api_create_sandbox.py b/packages/modal-infra/tests/test_web_api_create_sandbox.py index 48190f3228..496958280d 100644 --- a/packages/modal-infra/tests/test_web_api_create_sandbox.py +++ b/packages/modal-infra/tests/test_web_api_create_sandbox.py @@ -10,6 +10,7 @@ from sandbox_runtime.types import SandboxStatus from src import web_api from src.sandbox import manager as manager_module +from src.sandbox.launch_policy import DockerImageUnavailableError, InvalidDockerSettingsError from src.sandbox.manager import DEFAULT_SANDBOX_TIMEOUT_SECONDS @@ -39,6 +40,7 @@ async def create_sandbox(self, config): vnc_password=vnc_password, ttyd_url=None, tunnel_urls=None, + sandbox_backend="modal", ) monkeypatch.setattr(manager_module, "SandboxManager", FakeManager) @@ -64,6 +66,7 @@ async def restore_from_snapshot(self, **kwargs): vnc_password=vnc_password, ttyd_url=None, tunnel_urls=None, + sandbox_backend="modal", ) monkeypatch.setattr(manager_module, "SandboxManager", FakeManager) @@ -842,3 +845,72 @@ def test_session_config_helper_ignores_null_wire_values(): assert config.provider == "anthropic" assert config.model == "claude-sonnet-4-6" assert config.branch is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("call", "payload", "captured_key", "config_reader"), + [ + ( + _call_create_sandbox, + CREATE_REQUEST, + "config", + lambda captured: (captured["config"].settings, captured["config"].retire_sandbox_id), + ), + ( + _call_restore_sandbox, + RESTORE_REQUEST, + "restore", + lambda captured: ( + captured["restore"]["settings"], + captured["restore"]["retire_sandbox_id"], + ), + ), + ], +) +async def test_sandbox_requests_forward_docker_settings_and_report_the_launch( + monkeypatch, call, payload, captured_key, config_reader +): + _patch_auth(monkeypatch) + captured: dict = {} + if captured_key == "config": + _patch_manager(monkeypatch, captured) + else: + _patch_restore_manager(monkeypatch, captured) + settings = {"cpuCores": 2, "memoryMib": 4096} + + result = await call( + {**payload, "sandbox_settings": settings, "retire_sandbox_id": "sandbox-prior"} + ) + + assert config_reader(captured) == (settings, "sandbox-prior") + # The fake handle launched standard; the response reports what actually happened. + assert result["data"]["sandbox_backend"] == "modal" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("error", "status", "detail"), + [ + ( + InvalidDockerSettingsError("dockerEnabled must be a boolean"), + 400, + "dockerEnabled must be a boolean", + ), + (DockerImageUnavailableError("not provisioned"), 501, "docker_not_available"), + ], +) +async def test_docker_launch_errors_map_to_actionable_statuses(monkeypatch, error, status, detail): + _patch_auth(monkeypatch) + + class FailingManager: + async def create_sandbox(self, config): + raise error + + monkeypatch.setattr(manager_module, "SandboxManager", FailingManager) + + with pytest.raises(HTTPException) as exc_info: + await _call_create_sandbox(CREATE_REQUEST) + + assert exc_info.value.status_code == status + assert exc_info.value.detail == detail diff --git a/packages/sandbox-images/install/docker-daemon.json b/packages/sandbox-images/install/docker-daemon.json new file mode 100644 index 0000000000..3ebd8cdd1c --- /dev/null +++ b/packages/sandbox-images/install/docker-daemon.json @@ -0,0 +1,19 @@ +{ + "storage-driver": "overlay2", + "features": { + "containerd-snapshotter": false + }, + "bip": "10.200.0.1/24", + "default-address-pools": [ + { + "base": "10.201.0.0/16", + "size": 24 + } + ], + "shutdown-timeout": 20, + "log-driver": "local", + "log-opts": { + "max-size": "10m", + "max-file": "3" + } +} diff --git a/packages/sandbox-images/install/docker.sh b/packages/sandbox-images/install/docker.sh new file mode 100644 index 0000000000..74657604ea --- /dev/null +++ b/packages/sandbox-images/install/docker.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Optional Docker variant: adds a pinned Docker Engine, CLI, containerd, Buildx +# and Compose on top of the default image. Runs only while constructing the +# Docker image candidate (install.sh docker), never at session boot. +set -euo pipefail +source "$OI_INSTALL_DIR/common.sh" +if [[ "$OI_OS" != debian ]]; then + echo "The Docker variant currently requires Debian bookworm (got $OI_OS)" >&2 + exit 1 +fi +export DEBIAN_FRONTEND=noninteractive +package_dir="$(mktemp -d)" +rootfs="$(mktemp -d)" +trap 'rm -rf "$package_dir" "$rootfs"' EXIT +for tool in ENGINE CLI CONTAINERD BUILDX COMPOSE; do + file_key="DOCKER_${tool}_FILE" + hash_key="DOCKER_${tool}_SHA256" + download_checked \ + "https://download.docker.com/linux/debian/dists/bookworm/pool/stable/amd64/${!file_key}" \ + "${!hash_key}" \ + "$package_dir/${!file_key}" +done +apt-get update +apt-get install -y --no-install-recommends busybox-static "$package_dir"/*.deb +install -d -m 0755 /etc/docker +install -m 0644 "$OI_INSTALL_DIR/docker-daemon.json" /etc/docker/daemon.json +install -m 0644 "$OI_BUNDLE/packages/sandbox-images/verify/docker_smoke.py" /app/verify/docker_smoke.py +# A registry-independent BusyBox root filesystem lets image verification run +# a real build, container and Compose network without pulling anything. +mkdir -p "$rootfs/bin" +cp /bin/busybox "$rootfs/bin/busybox" +for applet in sh cat mkdir httpd wget sleep grep; do ln -s busybox "$rootfs/bin/$applet"; done +install -d -m 0755 /opt/openinspect/docker-smoke +tar --format=ustar --owner=0 --group=0 --numeric-owner -C "$rootfs" -cf /opt/openinspect/docker-smoke/rootfs.tar . +docker --version +docker buildx version +docker compose version diff --git a/packages/sandbox-images/src/sandbox_images/bundle.py b/packages/sandbox-images/src/sandbox_images/bundle.py index d42a1c7bfa..aad948f5d7 100644 --- a/packages/sandbox-images/src/sandbox_images/bundle.py +++ b/packages/sandbox-images/src/sandbox_images/bundle.py @@ -15,6 +15,7 @@ from .configuration import IMAGE_PACKAGE, RUNTIME_PACKAGE, read_json, runtime_environment from .locks import update_locks +DOCKER_PACKAGES = ("engine", "cli", "containerd", "buildx", "compose") PROVIDERS = ("modal", "daytona", "e2b", "vercel", "opencomputer") EXCLUDED = { ".terraform", @@ -99,6 +100,18 @@ def version(value: str) -> tuple[int, ...]: version(pin["version"]) if not re.fullmatch(r"[a-f0-9]{64}", pin["sha256"]): raise ValueError("Downloaded image tools must have a SHA-256 pin") + # The optional Docker variant installs pinned Debian packages, never a + # convenience script or an unpinned apt repository. + docker = tools.get("docker") + if not isinstance(docker, dict) or set(docker) != set(DOCKER_PACKAGES): + raise ValueError( + "Docker packages must pin exactly the engine, cli, containerd, buildx and compose" + ) + for pin in docker.values(): + if not re.fullmatch(r"[a-z0-9_.~+-]+_amd64\.deb", pin.get("file", "")): + raise ValueError("Docker packages must be pinned Debian amd64 artifacts") + if not re.fullmatch(r"[a-f0-9]{64}", pin.get("sha256", "")): + raise ValueError("Docker packages must have a SHA-256 pin") def source_files(root: Path, paths: tuple[Path, ...]) -> list[Path]: @@ -210,6 +223,10 @@ def pack_bundle(root: Path, provider: str, output_root: Path) -> PackedBundle: pin = toolchain[key][plan["target"]["node"]] if key == "node" else toolchain[key] variables[f"{name}_VERSION"] = pin["version"] variables[f"{name}_SHA256"] = pin["sha256"] + for package in DOCKER_PACKAGES: + pin = toolchain["docker"][package] + variables[f"DOCKER_{package.upper()}_FILE"] = pin["file"] + variables[f"DOCKER_{package.upper()}_SHA256"] = pin["sha256"] (destination / "image-config.sh").write_text( "\n".join(f"export {key}={shlex.quote(value)}" for key, value in variables.items()) + "\n" diff --git a/packages/sandbox-images/tests/test_bundle.py b/packages/sandbox-images/tests/test_bundle.py index 7ee324c120..182277c751 100644 --- a/packages/sandbox-images/tests/test_bundle.py +++ b/packages/sandbox-images/tests/test_bundle.py @@ -1,6 +1,7 @@ """Shared payload staging and conservative build invalidation.""" import json +import re import shutil from pathlib import Path @@ -146,3 +147,63 @@ def test_symlinks_and_executable_modes(checkout, tmp_path): (directory / "escape.sh").symlink_to(tmp_path / "outside") with pytest.raises(ValueError, match="symlink"): plan_image(checkout, "e2b") + + +def test_docker_variant_pins_debian_packages_by_checksum(): + tools = json.loads((REPO_ROOT / "packages/sandbox-images/toolchain.json").read_text()) + assert set(tools["docker"]) == {"engine", "cli", "containerd", "buildx", "compose"} + tools["docker"]["engine"]["file"] = "docker-ce_latest.tgz" + with pytest.raises(ValueError, match="Debian amd64"): + validate_toolchain(tools) + tools = json.loads((REPO_ROOT / "packages/sandbox-images/toolchain.json").read_text()) + tools["docker"]["compose"]["sha256"] = "deadbeef" + with pytest.raises(ValueError, match="SHA-256"): + validate_toolchain(tools) + del tools["docker"]["buildx"] + with pytest.raises(ValueError, match="exactly"): + validate_toolchain(tools) + + +def test_docker_variant_installs_only_checked_packages_and_never_runs_at_boot(): + script = (REPO_ROOT / "packages/sandbox-images/install/docker.sh").read_text() + assert "download_checked" in script + assert "get.docker.com" not in script + assert "docker pull" not in script + install_commands = [line for line in script.splitlines() if "apt-get install" in line] + assert install_commands + assert all("--no-install-recommends" in command for command in install_commands) + assert "/etc/docker/daemon.json" in script + # The default image's installer never includes the Docker phase. + install = (REPO_ROOT / "packages/sandbox-images/install/install.sh").read_text() + assert "docker" not in install + + +def test_docker_daemon_config_uses_overlay2_and_avoids_modal_networks(): + config = json.loads( + (REPO_ROOT / "packages/sandbox-images/install/docker-daemon.json").read_text() + ) + assert config["storage-driver"] == "overlay2" + assert config["features"]["containerd-snapshotter"] is False + assert config["bip"].startswith("10.") + assert all(pool["base"].startswith("10.") for pool in config["default-address-pools"]) + + +def test_docker_pins_reach_the_installer_as_shell_variables(checkout, tmp_path): + bundle = pack_bundle(checkout, "modal", tmp_path / "out") + config = (bundle.directory / "image-config.sh").read_text() + for package in ("ENGINE", "CLI", "CONTAINERD", "BUILDX", "COMPOSE"): + assert f"export DOCKER_{package}_FILE=" in config + assert f"export DOCKER_{package}_SHA256=" in config + + +def test_docker_daemon_shutdown_timeout_fits_inside_the_runtime_stop_deadline(): + config = json.loads( + (REPO_ROOT / "packages/sandbox-images/install/docker-daemon.json").read_text() + ) + runtime = ( + REPO_ROOT / "packages/sandbox-runtime/src/sandbox_runtime/docker_service.py" + ).read_text() + stop_deadline = float( + re.search(r"^DOCKER_STOP_TIMEOUT_SECONDS = ([0-9.]+)", runtime, re.M).group(1) + ) + assert config["shutdown-timeout"] < stop_deadline diff --git a/packages/sandbox-images/toolchain.json b/packages/sandbox-images/toolchain.json index 5aaef42579..15269adf52 100644 --- a/packages/sandbox-images/toolchain.json +++ b/packages/sandbox-images/toolchain.json @@ -1,6 +1,28 @@ { "schemaVersion": 1, "osRefresh": "2026-09-06", + "docker": { + "engine": { + "file": "docker-ce_29.8.1-1~debian.12~bookworm_amd64.deb", + "sha256": "3e38704938b78358563164cf419d00598e32a2119efc7cea80bdec2249efaf63" + }, + "cli": { + "file": "docker-ce-cli_29.8.1-1~debian.12~bookworm_amd64.deb", + "sha256": "ff812c5853c52ef120ec73132320805d179a376e42785085e2053ce7f2479860" + }, + "containerd": { + "file": "containerd.io_2.3.5-1~debian.12~bookworm_amd64.deb", + "sha256": "f821adc67b4bd1bc8da06288ec3aa81e029cbb264bd6db569c7cdab7f8bb0701" + }, + "buildx": { + "file": "docker-buildx-plugin_0.37.1-1~debian.12~bookworm_amd64.deb", + "sha256": "cf36da1a4c31287fef2c0cb5413aa660f56d4ec183b4d24dc47cd83ebef0a2c6" + }, + "compose": { + "file": "docker-compose-plugin_5.5.1-1~debian.12~bookworm_amd64.deb", + "sha256": "0455e6a54bcc07b1bcb5d92d9e6e77dedc485433c87c1b75572e6d837a96e459" + } + }, "chrome": { "version": "152.0.7977.82", "sha256": "0704631fb3e4f741092e08f55272f90abc3e307f991f05f332924364415b02e0" diff --git a/packages/sandbox-images/verify/docker_smoke.py b/packages/sandbox-images/verify/docker_smoke.py new file mode 100644 index 0000000000..177c89bd66 --- /dev/null +++ b/packages/sandbox-images/verify/docker_smoke.py @@ -0,0 +1,153 @@ +"""Verify the Docker image variant on the runtime it will be launched with. + +Runs inside the candidate Docker image on a Modal VM, using the production +DockerService as the daemon owner. It proves the daemon reaches readiness on +overlay2, that a build, a container and a Compose network work without any +registry, and that the daemon stops cleanly the way an image build requires. +""" + +from __future__ import annotations + +import asyncio +import os +import sys +import tempfile +from pathlib import Path + +from sandbox_runtime.docker_service import DockerService +from sandbox_runtime.log_config import get_logger +from sandbox_runtime.process_output import spawn_owned_subprocess, terminate_owned_subprocess + +COMMAND_TIMEOUT_SECONDS = 120 +ROOTFS_TAR = "/opt/openinspect/docker-smoke/rootfs.tar" +ROOTFS_IMAGE = "openinspect/docker-smoke-rootfs:local" +BUILT_IMAGE = "openinspect/docker-smoke:local" +COMPOSE_PROJECT = "openinspect-docker-smoke" +MARKER = "compose-network-ok" + + +async def run_command(*command: str, expect: bytes | None = None) -> bytes: + process = await spawn_owned_subprocess( + asyncio.create_subprocess_exec( + *command, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.DEVNULL, + start_new_session=True, + ) + ) + try: + async with asyncio.timeout(COMMAND_TIMEOUT_SECONDS): + stdout, _ = await process.communicate() + except BaseException: + await terminate_owned_subprocess(process) + raise + if process.returncode != 0: + raise RuntimeError(f"Docker verification command failed: {command[1]}") + if expect is not None and stdout.strip() != expect: + raise RuntimeError(f"Docker verification command returned unexpected output: {command[1]}") + return stdout + + +def write_fixtures(workdir: Path) -> None: + (workdir / "Dockerfile").write_text( + f"FROM {ROOTFS_IMAGE}\nRUN mkdir -p /www && printf {MARKER} > /www/marker\n" + ) + (workdir / "compose.yaml").write_text( + "services:\n" + " server:\n" + f" image: {BUILT_IMAGE}\n" + " pull_policy: never\n" + ' command: ["httpd", "-f", "-p", "8080", "-h", "/www"]\n' + " client:\n" + f" image: {BUILT_IMAGE}\n" + " pull_policy: never\n" + " depends_on: [server]\n" + ' command: ["sh", "-c", "for i in 1 2 3 4 5 6 7 8 9 10; do ' + f'wget -qO- http://server:8080/marker | grep -qx {MARKER} && exit 0; sleep 1; done; exit 1"]\n' + ) + + +async def verify_workloads(workdir: Path) -> None: + await run_command("docker", "info", "--format", "{{.Driver}}", expect=b"overlay2") + await run_command("docker", "import", ROOTFS_TAR, ROOTFS_IMAGE) + await run_command( + "docker", "buildx", "build", "--pull=false", "--load", "--tag", BUILT_IMAGE, str(workdir) + ) + await run_command( + "docker", + "run", + "--pull=never", + "--rm", + BUILT_IMAGE, + "cat", + "/www/marker", + expect=MARKER.encode(), + ) + await run_command( + "docker", + "compose", + "--project-name", + COMPOSE_PROJECT, + "-f", + str(workdir / "compose.yaml"), + "up", + "--pull", + "never", + "--abort-on-container-exit", + "--exit-code-from", + "client", + ) + + +async def cleanup(workdir: Path) -> list[Exception]: + """Best-effort removal of everything the checks created; failures are returned, not raised.""" + failures: list[Exception] = [] + for command in ( + ( + "docker", + "compose", + "--project-name", + COMPOSE_PROJECT, + "-f", + str(workdir / "compose.yaml"), + "down", + "--volumes", + "--remove-orphans", + ), + ("docker", "image", "rm", "--force", BUILT_IMAGE, ROOTFS_IMAGE), + ): + try: + await run_command(*command) + except Exception as error: + failures.append(error) + return failures + + +async def main() -> int: + service = DockerService(get_logger("docker-smoke")) + await service.start() + try: + with tempfile.TemporaryDirectory() as directory: + workdir = Path(directory) + write_fixtures(workdir) + try: + await verify_workloads(workdir) + except BaseException: + # The verification failure is the diagnosis; cleanup errors are secondary. + await cleanup(workdir) + raise + failures = await cleanup(workdir) + if failures: + raise RuntimeError("Docker verification cleanup failed") from failures[0] + # The build path stops the daemon the same way before a snapshot. + await service.prepare_for_snapshot() + finally: + await service.stop() + return 0 + + +if __name__ == "__main__": + if os.geteuid() != 0: + print("Docker verification must run as root", file=sys.stderr) + sys.exit(1) + sys.exit(asyncio.run(main())) diff --git a/packages/sandbox-runtime/src/sandbox_runtime/bridge.py b/packages/sandbox-runtime/src/sandbox_runtime/bridge.py index 91f146bfd1..80693380d3 100644 --- a/packages/sandbox-runtime/src/sandbox_runtime/bridge.py +++ b/packages/sandbox-runtime/src/sandbox_runtime/bridge.py @@ -46,9 +46,11 @@ from .boot_attach import RECONNECT_BACKOFF_BASE, RECONNECT_MAX_DELAY_SECONDS, BootAttach from .constants import ( BRIDGE_FATAL_ERROR_FILE_PATH, + DOCKER_ENABLED_ENV_VAR, REPO_MANIFEST_FILE_PATH, ) from .diff_capture import ControlPlaneDiffClient, SessionDiffRefreshWorker +from .docker_control import request as request_docker_preparation from .event_forwarder import BufferedEventForwarder from .git_signing import GitSigningError, GitSigningRuntime from .harness import ( @@ -882,6 +884,8 @@ async def contain_activity(deadline: float) -> bool: async def persist_session() -> None: await self._persist_rotated_session_id(self._require_harness(), strict=True) + if os.environ.get(DOCKER_ENABLED_ENV_VAR) == "true": + await request_docker_preparation("prepare") result = await self.shutdown_preparation.prepare( cmd, diff --git a/packages/sandbox-runtime/src/sandbox_runtime/constants.py b/packages/sandbox-runtime/src/sandbox_runtime/constants.py index 48227da6f3..633f2b0916 100644 --- a/packages/sandbox-runtime/src/sandbox_runtime/constants.py +++ b/packages/sandbox-runtime/src/sandbox_runtime/constants.py @@ -3,6 +3,10 @@ # Provider-selected directory for standalone runtime commands. OpenComputer uses # the sandbox user's bin directory; providers with writable images use /usr/local/bin. BIN_INSTALL_DIR_ENV_VAR = "OPENINSPECT_BIN_INSTALL_DIR" + +# Trusted launch signal: the provider sets it to "true" only for a sandbox it +# launched on a Docker-capable runtime. User env vars can never set it. +DOCKER_ENABLED_ENV_VAR = "OPENINSPECT_DOCKER_ENABLED" DEFAULT_BIN_INSTALL_DIR = "/usr/local/bin" # Sandbox lifetime and the env contract used to pass it to the bridge. diff --git a/packages/sandbox-runtime/src/sandbox_runtime/docker_control.py b/packages/sandbox-runtime/src/sandbox_runtime/docker_control.py new file mode 100644 index 0000000000..6c1294759a --- /dev/null +++ b/packages/sandbox-runtime/src/sandbox_runtime/docker_control.py @@ -0,0 +1,79 @@ +"""Local, terminal Docker preparation owned by the runtime supervisor. + +VM snapshots are terminal: only acknowledged preparation permits capture. +The provider retires the VM after capture; there is no implicit container restart. +""" + +from __future__ import annotations + +import asyncio +import sys +from pathlib import Path +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from .docker_service import DockerService + +SOCKET_PATH = "/tmp/openinspect-docker-control.sock" +CONTROL_TIMEOUT_SECONDS = 45 + + +class DockerControl: + def __init__(self, service: DockerService, path: str = SOCKET_PATH) -> None: + self.service = service + self.path = path + self.prepared = False + self._lock = asyncio.Lock() + self._server: asyncio.Server | None = None + + async def start(self) -> None: + Path(self.path).unlink(missing_ok=True) + self._server = await asyncio.start_unix_server(self._handle, path=self.path) + Path(self.path).chmod(0o600) + + async def stop(self) -> None: + if self._server: + self._server.close() + await self._server.wait_closed() + Path(self.path).unlink(missing_ok=True) + + async def _handle(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + try: + async with asyncio.timeout(CONTROL_TIMEOUT_SECONDS): + command = await reader.readline() + async with self._lock: + if command == b"prepare\n" and not self.prepared: + await self.service.prepare_for_snapshot() + self.prepared = True + result = ( + b"prepared\n" + if self.prepared and command in (b"prepare\n", b"status\n") + else b"not_prepared\n" + ) + writer.write(result) + await writer.drain() + except (Exception, asyncio.CancelledError): + # No acknowledgement is a failure, never permission to capture. + pass + finally: + writer.close() + await writer.wait_closed() + + +async def request(command: str, path: str = SOCKET_PATH) -> None: + async with asyncio.timeout(CONTROL_TIMEOUT_SECONDS): + reader, writer = await asyncio.open_unix_connection(path) + try: + writer.write((command + "\n").encode()) + await writer.drain() + if await reader.readline() != b"prepared\n": + raise RuntimeError("Docker snapshot requires confirmed shutdown preparation") + finally: + writer.close() + await writer.wait_closed() + + +if __name__ == "__main__": + if len(sys.argv) != 2 or sys.argv[1] not in ("status", "prepare"): + raise SystemExit(2) + asyncio.run(request(sys.argv[1])) diff --git a/packages/sandbox-runtime/src/sandbox_runtime/docker_service.py b/packages/sandbox-runtime/src/sandbox_runtime/docker_service.py new file mode 100644 index 0000000000..c603f6e789 --- /dev/null +++ b/packages/sandbox-runtime/src/sandbox_runtime/docker_service.py @@ -0,0 +1,204 @@ +"""Foreground-supervised Docker daemon for Docker-enabled sandboxes. + +The runtime supervisor owns ``dockerd`` as an ordinary child process: it is +started before repository hooks, watched for the life of the session, stopped +cleanly before an image build reports success, and reaped on shutdown. Nothing +backgrounds or detaches it, and no remote exec ever owns it. + +Raw daemon and probe output is deliberately not forwarded into the structured +log: registry errors and container metadata can carry repository secrets. The +daemon writes to a log file inside the sandbox for users; the runtime emits +only fixed lifecycle events. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import os +import signal +from pathlib import Path +from typing import Any + +from .process_output import ( + finish_cancellation_cleanup, + spawn_owned_subprocess, + terminate_owned_subprocess, + wait_for_process_exit, +) + +DOCKER_SOCKET = "unix:///var/run/docker.sock" +DOCKER_LOG_PATH = "/var/log/dockerd.log" +DOCKER_START_TIMEOUT_SECONDS = 60.0 +DOCKER_PROBE_TIMEOUT_SECONDS = 5.0 +DOCKER_PROBE_INTERVAL_SECONDS = 0.2 +DOCKER_STOP_TIMEOUT_SECONDS = 30.0 + + +class DockerService: + """Own one ``dockerd`` process from start through clean stop.""" + + def __init__( + self, + log: Any, + *, + start_timeout_seconds: float = DOCKER_START_TIMEOUT_SECONDS, + stop_timeout_seconds: float = DOCKER_STOP_TIMEOUT_SECONDS, + log_path: str = DOCKER_LOG_PATH, + ) -> None: + self.log = log + self.start_timeout_seconds = start_timeout_seconds + self.stop_timeout_seconds = stop_timeout_seconds + self.log_path = log_path + self._process: asyncio.subprocess.Process | None = None + self._exit_expected = False + self._preparation_finished = asyncio.Event() + self._preparation_finished.set() + + @property + def exit_expected(self) -> bool: + """Whether the exit is confirmed preparation or deliberate supervisor teardown.""" + return self._exit_expected + + async def start(self) -> None: + """Start the daemon and wait, under a deadline, until ``docker info`` succeeds.""" + if self._process is not None: + raise RuntimeError("Docker service already started") + self._exit_expected = False + self._process = await self._spawn_daemon() + try: + async with asyncio.timeout(self.start_timeout_seconds): + await self._wait_until_ready() + except TimeoutError: + await self.stop() + raise RuntimeError( + "Required Docker daemon did not become ready before its startup deadline" + ) from None + except BaseException: + await self.stop() + raise + self.log.info("docker.ready") + + async def _spawn_daemon(self) -> asyncio.subprocess.Process: + # The daemon's own output goes to a file the user can read inside the + # sandbox; a closed pipe must never be what stops the daemon. + daemon_log = open(self.log_path, "ab") # noqa: SIM115 - handed to the child + try: + return await spawn_owned_subprocess( + asyncio.create_subprocess_exec( + "dockerd", + "--host", + DOCKER_SOCKET, + stdin=asyncio.subprocess.DEVNULL, + stdout=daemon_log, + stderr=asyncio.subprocess.STDOUT, + start_new_session=True, + ) + ) + finally: + daemon_log.close() + + async def _wait_until_ready(self) -> None: + process = self._process + assert process is not None + while process.returncode is None: + if await self._probe() and process.returncode is None: + return + await asyncio.sleep(DOCKER_PROBE_INTERVAL_SECONDS) + raise RuntimeError("Required Docker daemon exited during startup") + + async def _probe(self) -> bool: + """One bounded ``docker info``; its output is discarded, only the exit code counts.""" + probe = await spawn_owned_subprocess( + asyncio.create_subprocess_exec( + "docker", + "--host", + DOCKER_SOCKET, + "info", + env={ + "PATH": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "HOME": "/root", + }, + stdin=asyncio.subprocess.DEVNULL, + stdout=asyncio.subprocess.DEVNULL, + stderr=asyncio.subprocess.DEVNULL, + start_new_session=True, + ) + ) + try: + async with asyncio.timeout(DOCKER_PROBE_TIMEOUT_SECONDS): + return await wait_for_process_exit(probe) == 0 + except TimeoutError: + return False + finally: + if probe.returncode is None: + cleanup = asyncio.create_task(terminate_owned_subprocess(probe)) + await finish_cancellation_cleanup(cleanup) + + async def wait(self) -> int: + """Block until the daemon exits and return its exit code.""" + process = self._process + if process is None: + raise RuntimeError("Docker service is not running") + exit_code = await wait_for_process_exit(process) + await self._preparation_finished.wait() + return exit_code + + async def prepare_for_snapshot(self) -> None: + """Stop the daemon cleanly so an image build can be snapshotted. + + Only the daemon leader is signalled: Docker must order container and + containerd shutdown itself before its own clean exit, so the whole + group is not killed. A non-zero exit, a timeout, or a daemon that was + already gone is a failed preparation, never a successful build. + """ + process = self._process + if process is None or process.returncode is not None: + raise RuntimeError("Required Docker daemon exited before build preparation") + self._preparation_finished.clear() + try: + process.send_signal(signal.SIGTERM) + try: + async with asyncio.timeout(self.stop_timeout_seconds): + if await wait_for_process_exit(process) != 0: + raise RuntimeError("Docker build preparation did not stop cleanly") + except TimeoutError: + await self.stop() + raise RuntimeError( + "Docker build preparation exceeded its clean shutdown deadline" + ) from None + # A clean daemon exit means it already stopped containerd and BuildKit; + # anything still alive in the group is a straggler, not a dependency. + with contextlib.suppress(ProcessLookupError): + os.killpg(process.pid, signal.SIGKILL) + self._process = None + # A reusable image must never include secret-bearing daemon diagnostics. + Path(self.log_path).write_bytes(b"") + self.log.info("docker.prepared") + self._exit_expected = True + except BaseException: + self._exit_expected = False + raise + finally: + self._preparation_finished.set() + + async def stop(self) -> None: + """Bounded graceful termination, then reap every owned process.""" + process = self._process + self._process = None + self._exit_expected = True + if process is None: + return + + async def terminate() -> None: + if process.returncode is None: + with contextlib.suppress(ProcessLookupError): + process.send_signal(signal.SIGTERM) + with contextlib.suppress(TimeoutError): + async with asyncio.timeout(self.stop_timeout_seconds): + await wait_for_process_exit(process) + await terminate_owned_subprocess(process) + + cleanup = asyncio.create_task(terminate()) + await finish_cancellation_cleanup(cleanup) + self.log.info("docker.stopped") diff --git a/packages/sandbox-runtime/src/sandbox_runtime/entrypoint.py b/packages/sandbox-runtime/src/sandbox_runtime/entrypoint.py index 035e53ca7b..a018007288 100644 --- a/packages/sandbox-runtime/src/sandbox_runtime/entrypoint.py +++ b/packages/sandbox-runtime/src/sandbox_runtime/entrypoint.py @@ -15,6 +15,7 @@ from .claude_stager import ClaudeStager, isolated_claude_config_dir, resolve_claude_config_dir from .code_server import CodeServer from .constants import VNC_DISPLAY, VNC_PASSWORD_ENV_VAR +from .docker_service import DockerService from .harness.base import HarnessId, HarnessProcessOwner from .image_build_context_start import ( IMAGE_BUILD_CONTEXT_START_ARGUMENT, @@ -146,6 +147,7 @@ def build_supervisor(shutdown_event: asyncio.Event) -> SandboxSupervisor: shutdown_event, log, boot_events=warnings, + docker_service=DockerService(log) if config.docker_enabled else None, ) diff --git a/packages/sandbox-runtime/src/sandbox_runtime/image_build_context_start.py b/packages/sandbox-runtime/src/sandbox_runtime/image_build_context_start.py index 8972c91763..50e3471ebb 100644 --- a/packages/sandbox-runtime/src/sandbox_runtime/image_build_context_start.py +++ b/packages/sandbox-runtime/src/sandbox_runtime/image_build_context_start.py @@ -32,7 +32,7 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Protocol -from .constants import IMAGE_BUILD_EXECUTION_TIMEOUT_ENV_VAR +from .constants import DOCKER_ENABLED_ENV_VAR, IMAGE_BUILD_EXECUTION_TIMEOUT_ENV_VAR from .log_config import StructuredLogger, get_logger from .repo_image_callback import ( BUILD_ID_ENV, @@ -101,6 +101,8 @@ "VCS_HOST", "VCS_CLONE_USERNAME", VCS_CLONE_TOKEN_ENV, + # Trusted runtime signal (provider-owned). + DOCKER_ENABLED_ENV_VAR, } ) diff --git a/packages/sandbox-runtime/src/sandbox_runtime/runtime_config.py b/packages/sandbox-runtime/src/sandbox_runtime/runtime_config.py index 4733d92ded..92214a6071 100644 --- a/packages/sandbox-runtime/src/sandbox_runtime/runtime_config.py +++ b/packages/sandbox-runtime/src/sandbox_runtime/runtime_config.py @@ -11,6 +11,7 @@ from typing import Any from urllib.parse import urlsplit +from .constants import DOCKER_ENABLED_ENV_VAR from .harness.base import HarnessId, parse_harness_id @@ -110,6 +111,9 @@ class RuntimeConfig: session_config: Mapping[str, Any] workspace_path: Path repo_path: Path + # Set by the provider, never by user configuration: this sandbox runs on a + # Docker-capable runtime and must own a daemon before repository hooks. + docker_enabled: bool = False @classmethod def from_env( @@ -137,6 +141,7 @@ def from_env( session_config=session_config, workspace_path=workspace_path, repo_path=repo_path, + docker_enabled=environment.get(DOCKER_ENABLED_ENV_VAR) == "true", ) @property diff --git a/packages/sandbox-runtime/src/sandbox_runtime/supervisor.py b/packages/sandbox-runtime/src/sandbox_runtime/supervisor.py index e6e4c0c6e4..fdf2f9c0f1 100644 --- a/packages/sandbox-runtime/src/sandbox_runtime/supervisor.py +++ b/packages/sandbox-runtime/src/sandbox_runtime/supervisor.py @@ -16,6 +16,7 @@ BRIDGE_FATAL_ERROR_FILE_PATH, IMAGE_BUILD_EXECUTION_TIMEOUT_ENV_VAR, ) +from .docker_control import DockerControl from .harness.base import DETERMINISTIC_FAILURE_EXIT_CODE from .repo_image_callback import RepoImageBuildCallback from .runtime_config import BootMode, RuntimeConfig @@ -28,6 +29,7 @@ from .boot_events import BootEventLog from .browser_desktop import BrowserDesktop from .code_server import CodeServer + from .docker_service import DockerService from .harness.base import HarnessProcessOwner from .managed_skills import ManagedSkillsMaterializer from .repository_boot import RepositoryBoot, RepositoryBootResult @@ -71,9 +73,16 @@ def __init__( log: Any, *, boot_events: BootEventLog | None = None, + docker_service: DockerService | None = None, ) -> None: self.config = config self.repository_boot = repository_boot + # Present only for Docker-enabled sandboxes: started before repository + # hooks, watched for the whole session, stopped last. + self.docker_service = docker_service + self.docker_control = DockerControl(docker_service) if docker_service is not None else None + self._docker_watch_task: asyncio.Task[None] | None = None + self._docker_watch_failure: BaseException | None = None # The boot-events channel the bridge relays; the repository boot # writes its own phases and warnings through the same log. self.boot_events: BootEventLog = ( @@ -379,6 +388,45 @@ async def _watch_bridge_during_boot(self) -> None: self._bridge_watch_failure = error self.shutdown_event.set() + async def _start_docker(self) -> None: + """Start the owned daemon; only called when the trusted launch config requires Docker.""" + if self.docker_service is None: + raise RuntimeError("Required Docker service is not configured") + await self.docker_service.start() + self._docker_watch_task = asyncio.create_task(self._watch_docker()) + if self.docker_control is not None and self.boot_mode is not BootMode.BUILD: + await self.docker_control.start() + + async def _watch_docker(self) -> None: + """An unrequested daemon exit is fatal for as long as Docker is required.""" + service = self.docker_service + assert service is not None + try: + await service.wait() + except asyncio.CancelledError: + raise + except Exception as error: + self.log.error("docker.watch_failed", exc=error) + self._docker_watch_failure = error + self.shutdown_event.set() + return + if service.exit_expected: + return + self.log.error("docker.exited_unexpectedly") + self._docker_watch_failure = RuntimeError("Required Docker daemon exited unexpectedly") + # Interrupt hooks and the process monitor; the failure is reported + # by ``run`` rather than treated as a requested shutdown. + self.shutdown_event.set() + + async def _stop_docker_watch(self) -> None: + task = self._docker_watch_task + if task is None: + return + self._docker_watch_task = None + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + async def _stop_bridge_watch(self) -> None: task = self._bridge_watch_task if task is None: @@ -451,7 +499,7 @@ def _boot_interruption(self) -> BaseException: A failure is raised as itself so ``run`` reports it fatally; a requested shutdown is a clean end to the boot. """ - failure = self._bridge_watch_failure + failure = self._docker_watch_failure or self._bridge_watch_failure if failure is not None: return failure return BootExecutionCancelled() @@ -508,9 +556,23 @@ async def _run_image_build_execution( timeout_seconds = self._image_build_execution_timeout_seconds() try: async with asyncio.timeout(timeout_seconds): - return await self._run_until_shutdown( + if not self.config.docker_enabled: + return await self._run_until_shutdown( + lambda: self.repository_boot.boot(BootMode.BUILD, expected_tunnel_ports) + ) + # Docker starts before setup hooks, and is stopped cleanly + # before success is reported: the snapshot must hold a + # quiesced data root, never a daemon mid-write. + await self._run_until_shutdown(self._start_docker) + result = await self._run_until_shutdown( lambda: self.repository_boot.boot(BootMode.BUILD, expected_tunnel_ports) ) + assert self.docker_service is not None + await self._run_until_shutdown(self.docker_service.prepare_for_snapshot) + await self._stop_docker_watch() + if self._docker_watch_failure is not None: + raise self._docker_watch_failure + return result except TimeoutError as error: raise RuntimeError( f"image build exceeded its {timeout_seconds}-second execution timeout" @@ -613,6 +675,11 @@ async def run(self, repo_image_callback: RepoImageBuildCallback | None = None) - await self.agent_bridge.start(early_connect=True) self._bridge_watch_task = asyncio.create_task(self._watch_bridge_during_boot()) + if self.config.docker_enabled: + # Docker before the desktop and the repository boot: setup and + # start hooks may run containers. + await self._run_until_shutdown(self._start_docker) + try: await self.browser_desktop.start() except Exception as error: @@ -655,6 +722,10 @@ async def run(self, repo_image_callback: RepoImageBuildCallback | None = None) - if self._bridge_watch_failure is not None: raise self._bridge_watch_failure await self.monitor_processes() + # The Docker watcher runs for the whole session: a daemon that + # died under a working harness ended the session as a failure. + if self._docker_watch_failure is not None: + raise self._docker_watch_failure except BootExecutionCancelled: event = ( "image_build.cancelled" @@ -665,23 +736,41 @@ async def run(self, repo_image_callback: RepoImageBuildCallback | None = None) - return True except Exception as error: self.log.error("supervisor.error", exc=error) - if self.boot_mode is BootMode.BUILD and self.shutdown_event.is_set(): + docker_failed = self._docker_watch_failure is not None + if ( + self.boot_mode is BootMode.BUILD + and self.shutdown_event.is_set() + and not docker_failed + ): self.log.info("image_build.cancelled", reason="shutdown_requested") return True if self.boot_mode is BootMode.BUILD and repo_image_callback: - try: - error_message = str(error) - await self._run_until_shutdown( - lambda: repo_image_callback.report_failure(error_message) - ) - except BootExecutionCancelled: - self.log.info("image_build.cancelled", reason="shutdown_requested") - return True + error_message = str(error) + if docker_failed: + # The watcher set shutdown_event to interrupt hooks; that + # is not a requested cancellation and the failure must + # still reach the control plane. + try: + # The callback owns its bounded retries; the daemon failure's + # shutdown signal must not cancel delivery. + if not await repo_image_callback.report_failure(error_message): + self.log.error("image_build.failure_report_failed") + except Exception: + self.log.error("image_build.failure_report_failed") + else: + try: + await self._run_until_shutdown( + lambda: repo_image_callback.report_failure(error_message) + ) + except BootExecutionCancelled: + self.log.info("image_build.cancelled", reason="shutdown_requested") + return True await self._report_fatal_error( str(error), error if isinstance(error, BootPhaseError) else None ) return False finally: + await self._stop_docker_watch() await self._stop_bridge_watch() await self.shutdown() return True @@ -692,6 +781,8 @@ def request_shutdown(self, sig: signal.Signals) -> None: async def shutdown(self) -> None: self.log.info("supervisor.shutdown_start") + if self.docker_control is not None: + await self.docker_control.stop() if self._desktop_restart_task and not self._desktop_restart_task.done(): self._desktop_restart_task.cancel() await asyncio.gather(self._desktop_restart_task, return_exceptions=True) @@ -701,4 +792,7 @@ async def shutdown(self) -> None: await self.code_server.stop() await self.browser_desktop.stop() await self.harness_process.stop() + # User containers outlive the harness that drove them, never the reverse. + if self.docker_service is not None: + await self.docker_service.stop() self.log.info("supervisor.shutdown_complete") diff --git a/packages/sandbox-runtime/tests/test_bridge_reconnection.py b/packages/sandbox-runtime/tests/test_bridge_reconnection.py index 3734af644e..757b7ff65d 100644 --- a/packages/sandbox-runtime/tests/test_bridge_reconnection.py +++ b/packages/sandbox-runtime/tests/test_bridge_reconnection.py @@ -43,6 +43,12 @@ def test_http_500_is_not_fatal(self, bridge): error_str = "server rejected WebSocket connection: HTTP 500" assert bridge._is_fatal_connection_error(error_str) is False + def test_http_503_is_not_fatal(self, bridge): + # The control plane answers 503 while a save still needs this sandbox; + # exiting would shut the sandbox down under the save. + error_str = "server rejected WebSocket connection: HTTP 503" + assert bridge._is_fatal_connection_error(error_str) is False + def test_network_error_is_not_fatal(self, bridge): error_str = "Connection refused" assert bridge._is_fatal_connection_error(error_str) is False diff --git a/packages/sandbox-runtime/tests/test_docker_control.py b/packages/sandbox-runtime/tests/test_docker_control.py new file mode 100644 index 0000000000..8dca7ea1f4 --- /dev/null +++ b/packages/sandbox-runtime/tests/test_docker_control.py @@ -0,0 +1,48 @@ +import asyncio +import tempfile +from unittest.mock import AsyncMock, Mock + +import pytest + +from sandbox_runtime.docker_control import DockerControl, request + + +@pytest.fixture +def socket_path(): + with tempfile.TemporaryDirectory(prefix="oi-docker-", dir="/tmp") as directory: + yield directory + "/control.sock" + + +@pytest.mark.asyncio +async def test_preparation_is_acknowledged_only_after_clean_stop(socket_path): + service = Mock(prepare_for_snapshot=AsyncMock()) + # Short paths also work on macOS's small Unix-domain path limit. + path = socket_path + control = DockerControl(service, path) + await control.start() + try: + with pytest.raises(RuntimeError, match="confirmed shutdown"): + await request("status", path) + await asyncio.gather(request("prepare", path), request("prepare", path)) + # A lost capture response may prepare the same retained VM again. + await request("prepare", path) + service.prepare_for_snapshot.assert_awaited_once() + await request("status", path) + finally: + await control.stop() + + +@pytest.mark.asyncio +async def test_failed_preparation_never_acknowledges_capture(socket_path): + service = Mock(prepare_for_snapshot=AsyncMock(side_effect=RuntimeError("stop failed"))) + path = socket_path + control = DockerControl(service, path) + await control.start() + try: + with pytest.raises(RuntimeError): + await request("prepare", path) + assert not control.prepared + with pytest.raises(RuntimeError): + await request("status", path) + finally: + await control.stop() diff --git a/packages/sandbox-runtime/tests/test_docker_service.py b/packages/sandbox-runtime/tests/test_docker_service.py new file mode 100644 index 0000000000..5d3d26df3f --- /dev/null +++ b/packages/sandbox-runtime/tests/test_docker_service.py @@ -0,0 +1,222 @@ +"""DockerService owns dockerd from start through clean stop, deterministically. + +The fakes implement a real readiness handshake: the fake daemon installs its +SIGTERM handler and only then publishes a ready marker, and the fake +``docker info`` succeeds only once that marker exists. The service therefore +cannot observe readiness before the daemon can honor a clean stop, which is +also the production contract (``docker info`` succeeds only once the daemon +serves the socket). +""" + +from __future__ import annotations + +import asyncio +import os +import sys +from dataclasses import dataclass, field + +import pytest + +from sandbox_runtime import docker_service as docker_module +from sandbox_runtime.docker_service import DockerService + + +@dataclass +class FakeProcesses: + ready_marker: str + daemon_exit: int = 0 + probe_delay: float = 0.0 + fail_probe: bool = False + ignore_sigterm: bool = False + children: list[asyncio.subprocess.Process] = field(default_factory=list) + spawns: list[tuple[str, ...]] = field(default_factory=list) + + def daemon_program(self) -> str: + handler = ( + "signal.SIG_IGN" if self.ignore_sigterm else f"lambda *_: sys.exit({self.daemon_exit})" + ) + return ( + "import signal, sys, time, pathlib; " + f"signal.signal(signal.SIGTERM, {handler}); " + f"pathlib.Path({self.ready_marker!r}).write_text('ready'); " + "time.sleep(300)" + ) + + def probe_program(self) -> str: + return ( + "import sys, time, pathlib; " + f"time.sleep({self.probe_delay}); " + f"sys.exit(1 if {self.fail_probe} or not pathlib.Path({self.ready_marker!r}).exists() else 0)" + ) + + +@pytest.fixture +def processes(monkeypatch, tmp_path): + fakes = FakeProcesses(ready_marker=str(tmp_path / "dockerd.ready")) + real_spawn = asyncio.create_subprocess_exec + + async def spawn(command, *args, **kwargs): + fakes.spawns.append((command, *args)) + assert kwargs.get("start_new_session") is True + if command == "dockerd": + program = fakes.daemon_program() + elif command == "docker": + assert kwargs["env"] == { + "PATH": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "HOME": "/root", + } + program = fakes.probe_program() + else: # pragma: no cover - the service spawns nothing else + raise AssertionError(command) + process = await real_spawn(sys.executable, "-c", program, **kwargs) + fakes.children.append(process) + return process + + monkeypatch.setattr(docker_module.asyncio, "create_subprocess_exec", spawn) + monkeypatch.setattr(docker_module, "DOCKER_PROBE_INTERVAL_SECONDS", 0.01) + yield fakes + for child in fakes.children: + if child.returncode is None: + child.kill() + + +def _service(tmp_path, **kwargs) -> DockerService: + class Log: + def __init__(self): + self.events: list[str] = [] + + def info(self, event, **_fields): + self.events.append(event) + + def error(self, event, **_fields): + self.events.append(event) + + service = DockerService(Log(), log_path=str(tmp_path / "dockerd.log"), **kwargs) + return service + + +def _group_gone(process: asyncio.subprocess.Process) -> bool: + try: + os.killpg(process.pid, 0) + except ProcessLookupError: + return True + return False + + +async def _until(predicate, timeout: float = 10.0) -> None: + """Bounded wait: a regression fails the test instead of hanging it.""" + async with asyncio.timeout(timeout): + while not predicate(): + await asyncio.sleep(0.01) + + +async def test_ready_then_clean_preparation_leaves_no_owned_process(processes, tmp_path): + service = _service(tmp_path) + + await service.start() + daemon = processes.children[0] + assert daemon.returncode is None + assert processes.spawns[0] == ("dockerd", "--host", "unix:///var/run/docker.sock") + assert processes.spawns[1] == ("docker", "--host", "unix:///var/run/docker.sock", "info") + + await service.prepare_for_snapshot() + + assert daemon.returncode == 0 + assert _group_gone(daemon) + assert service.exit_expected is True + await service.stop() + assert all(child.returncode is not None for child in processes.children) + + +async def test_startup_deadline_has_its_own_diagnostic_and_reaps_the_daemon(processes, tmp_path): + processes.fail_probe = True + service = _service(tmp_path, start_timeout_seconds=0.3) + + with pytest.raises(RuntimeError, match="startup deadline"): + await service.start() + + assert all(child.returncode is not None for child in processes.children) + assert all(_group_gone(child) for child in processes.children) + + +async def test_daemon_exit_during_startup_is_reported(processes, tmp_path): + processes.fail_probe = True + service = _service(tmp_path, start_timeout_seconds=60) + started = asyncio.create_task(service.start()) + await _until(lambda: bool(processes.children)) + processes.children[0].kill() + + with pytest.raises(RuntimeError, match="exited during startup"): + await started + + +async def test_cancellation_during_probe_reaps_both_process_groups(processes, tmp_path): + processes.probe_delay = 300 + service = _service(tmp_path, start_timeout_seconds=60) + started = asyncio.create_task(service.start()) + await _until(lambda: len(processes.children) >= 2) + + started.cancel() + with pytest.raises(asyncio.CancelledError): + await started + + assert all(child.returncode is not None for child in processes.children) + assert all(_group_gone(child) for child in processes.children) + + +async def test_nonzero_daemon_exit_cannot_be_a_prepared_build(processes, tmp_path): + processes.daemon_exit = 1 + service = _service(tmp_path) + await service.start() + watcher = asyncio.create_task(service.wait()) + await asyncio.sleep(0) + + with pytest.raises(RuntimeError, match="did not stop cleanly"): + await service.prepare_for_snapshot() + + assert await watcher == 1 + assert service.exit_expected is False + await service.stop() + assert all(child.returncode is not None for child in processes.children) + + +async def test_unexpected_exit_is_observable_and_not_a_requested_stop(processes, tmp_path): + service = _service(tmp_path) + await service.start() + daemon = processes.children[0] + + daemon.kill() + assert await service.wait() != 0 + assert service.exit_expected is False + + await service.stop() + assert service.exit_expected is True + + +async def test_daemon_that_ignores_sigterm_is_killed_and_never_a_prepared_build( + processes, tmp_path +): + processes.ignore_sigterm = True + service = _service(tmp_path, stop_timeout_seconds=0.3) + await service.start() + daemon = processes.children[0] + + with pytest.raises(RuntimeError, match="clean shutdown deadline"): + await service.prepare_for_snapshot() + + assert service.exit_expected is False + assert daemon.returncode is not None + assert _group_gone(daemon) + + +async def test_preparation_requires_a_running_daemon(processes, tmp_path): + service = _service(tmp_path) + with pytest.raises(RuntimeError, match="exited before build preparation"): + await service.prepare_for_snapshot() + + await service.start() + processes.children[0].kill() + await service.wait() + with pytest.raises(RuntimeError, match="exited before build preparation"): + await service.prepare_for_snapshot() + await service.stop() diff --git a/packages/sandbox-runtime/tests/test_runtime_config.py b/packages/sandbox-runtime/tests/test_runtime_config.py index b1f444baee..52e17aa5a6 100644 --- a/packages/sandbox-runtime/tests/test_runtime_config.py +++ b/packages/sandbox-runtime/tests/test_runtime_config.py @@ -78,3 +78,15 @@ def test_session_config_is_recursively_immutable(): assert isinstance(repositories[0], MappingProxyType) with pytest.raises(TypeError): repositories[0]["repo_name"] = "changed" + + +@pytest.mark.parametrize( + ("value", "expected"), + [(None, False), ("", False), ("false", False), ("1", False), ("TRUE", False), ("true", True)], +) +def test_docker_enabled_is_only_the_exact_trusted_value(value, expected): + environment = {"SANDBOX_ID": "sandbox-1"} + if value is not None: + environment["OPENINSPECT_DOCKER_ENABLED"] = value + + assert RuntimeConfig.from_env(environment).docker_enabled is expected diff --git a/packages/sandbox-runtime/tests/test_supervisor_lifecycle.py b/packages/sandbox-runtime/tests/test_supervisor_lifecycle.py index 788ed505b7..eec254d9a0 100644 --- a/packages/sandbox-runtime/tests/test_supervisor_lifecycle.py +++ b/packages/sandbox-runtime/tests/test_supervisor_lifecycle.py @@ -1,4 +1,5 @@ import asyncio +import signal from pathlib import Path from unittest.mock import AsyncMock, MagicMock @@ -284,3 +285,249 @@ async def test_code_server_restart_exhaustion_is_nonfatal(tmp_path, monkeypatch) await SandboxSupervisor.monitor_processes(supervisor) supervisor._report_fatal_error.assert_not_awaited() + + +def _docker_service(events, *, prepare_error=None): + service = MagicMock() + service.exit_expected = False + exited = asyncio.Event() + + async def start(): + events.append("docker:start") + + async def wait(): + await exited.wait() + return 137 + + async def prepare_for_snapshot(): + events.append("docker:prepare") + if prepare_error is not None: + raise prepare_error + service.exit_expected = True + + async def stop(): + events.append("docker:stop") + service.exit_expected = True + exited.set() + + service.start = AsyncMock(side_effect=start) + service.wait = AsyncMock(side_effect=wait) + service.prepare_for_snapshot = AsyncMock(side_effect=prepare_for_snapshot) + service.stop = AsyncMock(side_effect=stop) + service.exited = exited + return service + + +def _docker_supervisor(tmp_path, events, monkeypatch, **service_kwargs): + supervisor, repository, *rest = _supervisor(tmp_path, events) + supervisor.config = RuntimeConfig.from_env( + { + "SANDBOX_ID": "sandbox-1", + "REPO_OWNER": "acme", + "REPO_NAME": "repo", + "OPENINSPECT_DOCKER_ENABLED": "true", + }, + workspace_path=tmp_path, + ) + supervisor.docker_service = _docker_service(events, **service_kwargs) + return supervisor, repository, *rest + + +async def test_docker_starts_before_repository_boot_and_stops_last(tmp_path, monkeypatch): + events = [] + supervisor, *_ = _docker_supervisor(tmp_path, events, monkeypatch) + monkeypatch.delenv("IMAGE_BUILD_MODE", raising=False) + monkeypatch.delenv("RESTORED_FROM_SNAPSHOT", raising=False) + monkeypatch.delenv("FROM_REPO_IMAGE", raising=False) + + assert await supervisor.run() is True + + assert events[:3] == ["docker:start", "desktop", "repository:fresh"] + assert events[-1] == "docker:stop" + + +async def test_standard_boot_never_touches_docker(tmp_path, monkeypatch): + events = [] + supervisor, *_ = _supervisor(tmp_path, events) + monkeypatch.delenv("IMAGE_BUILD_MODE", raising=False) + monkeypatch.delenv("RESTORED_FROM_SNAPSHOT", raising=False) + monkeypatch.delenv("FROM_REPO_IMAGE", raising=False) + + assert await supervisor.run() is True + + assert supervisor.docker_service is None + assert not any(event.startswith("docker:") for event in events) + + +async def test_docker_required_but_unconfigured_is_fatal(tmp_path, monkeypatch): + events = [] + supervisor, *_ = _docker_supervisor(tmp_path, events, monkeypatch) + supervisor.docker_service = None + supervisor._report_fatal_error = AsyncMock() + monkeypatch.delenv("IMAGE_BUILD_MODE", raising=False) + + assert await supervisor.run() is False + + supervisor._report_fatal_error.assert_awaited_once() + assert "Docker service is not configured" in supervisor._report_fatal_error.await_args.args[0] + assert "repository:fresh" not in events + + +async def test_build_starts_docker_before_hooks_and_prepares_it_before_success( + tmp_path, monkeypatch +): + events = [] + supervisor, *_ = _docker_supervisor(tmp_path, events, monkeypatch) + monkeypatch.setenv("IMAGE_BUILD_MODE", "true") + callback = MagicMock() + + async def report_success(**_kwargs): + events.append("success") + assert supervisor._docker_watch_task is None + supervisor.shutdown_event.set() + return True + + callback.report_success = AsyncMock(side_effect=report_success) + callback.report_failure = AsyncMock() + + assert await supervisor.run(callback) is True + + assert events == [ + "docker:start", + "repository:build", + "docker:prepare", + "success", + "docker:stop", + ] + + +@pytest.mark.parametrize( + "error", [RuntimeError("did not stop cleanly"), RuntimeError("clean shutdown deadline")] +) +async def test_build_preparation_failure_is_reported_as_a_failed_build( + tmp_path, monkeypatch, error +): + events = [] + supervisor, *_ = _docker_supervisor(tmp_path, events, monkeypatch, prepare_error=error) + monkeypatch.setenv("IMAGE_BUILD_MODE", "true") + supervisor._report_fatal_error = AsyncMock() + callback = MagicMock() + callback.report_success = AsyncMock() + callback.report_failure = AsyncMock() + + assert await supervisor.run(callback) is False + + callback.report_success.assert_not_awaited() + callback.report_failure.assert_awaited_once() + assert callback.report_failure.await_args.args[0] + assert supervisor.docker_service.stop.await_count == 1 + + +@pytest.mark.parametrize("reported", [True, False]) +async def test_daemon_exit_during_build_hooks_fails_the_build(tmp_path, monkeypatch, reported): + events = [] + supervisor, repository, *_ = _docker_supervisor(tmp_path, events, monkeypatch) + monkeypatch.setenv("IMAGE_BUILD_MODE", "true") + supervisor._report_fatal_error = AsyncMock() + callback = MagicMock() + callback.report_success = AsyncMock() + callback.report_failure = AsyncMock() + # A zero fatal-report bound must not cancel the separate build callback policy. + monkeypatch.setattr("sandbox_runtime.supervisor.FATAL_ERROR_REPORT_TIMEOUT_SECONDS", 0) + + async def report_failure(_error): + await asyncio.sleep(0) + events.append("failure:reported") + return reported + + callback.report_failure.side_effect = report_failure + + async def boot(_mode, _ports): + events.append("repository:build") + supervisor.docker_service.exited.set() + await asyncio.Event().wait() + + repository.boot = AsyncMock(side_effect=boot) + + assert await supervisor.run(callback) is False + + callback.report_success.assert_not_awaited() + callback.report_failure.assert_awaited_once() + assert "exited unexpectedly" in callback.report_failure.await_args.args[0] + supervisor._report_fatal_error.assert_awaited_once() + assert "failure:reported" in events + if not reported: + supervisor.log.error.assert_any_call("image_build.failure_report_failed") + + +async def test_daemon_exit_during_session_is_fatal(tmp_path, monkeypatch): + events = [] + supervisor, *_ = _docker_supervisor(tmp_path, events, monkeypatch) + monkeypatch.delenv("IMAGE_BUILD_MODE", raising=False) + supervisor._report_fatal_error = AsyncMock() + + async def monitor(): + supervisor.docker_service.exited.set() + await supervisor.shutdown_event.wait() + + supervisor.monitor_processes = AsyncMock(side_effect=monitor) + + assert await supervisor.run() is False + + supervisor._report_fatal_error.assert_awaited_once() + assert "exited unexpectedly" in supervisor._report_fatal_error.await_args.args[0] + + +async def test_requested_shutdown_during_docker_start_is_not_a_failure(tmp_path, monkeypatch): + events = [] + supervisor, *_ = _docker_supervisor(tmp_path, events, monkeypatch) + monkeypatch.setenv("IMAGE_BUILD_MODE", "true") + callback = MagicMock() + callback.report_success = AsyncMock() + callback.report_failure = AsyncMock() + + async def start(): + supervisor.shutdown_event.set() + await asyncio.Event().wait() + + supervisor.docker_service.start = AsyncMock(side_effect=start) + + assert await supervisor.run(callback) is True + + callback.report_success.assert_not_awaited() + callback.report_failure.assert_not_awaited() + + +async def test_daemon_exit_during_interactive_boot_is_fatal(tmp_path, monkeypatch): + events = [] + supervisor, repository, *_ = _docker_supervisor(tmp_path, events, monkeypatch) + monkeypatch.delenv("IMAGE_BUILD_MODE", raising=False) + supervisor._report_fatal_error = AsyncMock() + + async def boot(_mode, _ports): + supervisor.docker_service.exited.set() + await asyncio.Event().wait() + + repository.boot = AsyncMock(side_effect=boot) + + assert await supervisor.run() is False + + supervisor._report_fatal_error.assert_awaited_once() + assert "exited unexpectedly" in supervisor._report_fatal_error.await_args.args[0] + + +async def test_requested_shutdown_with_docker_running_is_not_a_failure(tmp_path, monkeypatch): + events = [] + supervisor, *_ = _docker_supervisor(tmp_path, events, monkeypatch) + monkeypatch.delenv("IMAGE_BUILD_MODE", raising=False) + supervisor._report_fatal_error = AsyncMock() + + async def monitor(): + supervisor.request_shutdown(signal.SIGTERM) + + supervisor.monitor_processes = AsyncMock(side_effect=monitor) + + assert await supervisor.run() is True + + supervisor._report_fatal_error.assert_not_awaited() + assert events[-1] == "docker:stop" diff --git a/packages/shared/src/types/github-autofix.ts b/packages/shared/src/types/github-autofix.ts index ab2a85f2e8..5bfe77b0e8 100644 --- a/packages/shared/src/types/github-autofix.ts +++ b/packages/shared/src/types/github-autofix.ts @@ -145,7 +145,13 @@ export const githubAutofixSessionResponseSchema = z.discriminatedUnion("kind", [ }), z.object({ kind: z.literal("rejected"), - reason: z.enum(["session_closed", "budget_exhausted", "queue_full", "attempt_limit"]), + reason: z.enum([ + "session_closed", + "sandbox_recovery_required", + "budget_exhausted", + "queue_full", + "attempt_limit", + ]), }), z.object({ kind: z.literal("found"), diff --git a/packages/shared/src/types/image-builds.ts b/packages/shared/src/types/image-builds.ts index 54e70f1c75..049f75e0be 100644 --- a/packages/shared/src/types/image-builds.ts +++ b/packages/shared/src/types/image-builds.ts @@ -8,6 +8,19 @@ */ import { z } from "zod"; +import type { SandboxProviderName } from "./integrations"; + +/** Backends that can build and boot prebuilt images. + * Deployment admission (such as Daytona opt-in) is a separate policy. + */ +export const IMAGE_BUILD_PROVIDER_IDS = [ + "modal", + "modal-vm", + "vercel", + "opencomputer", + "e2b", + "daytona", +] as const satisfies readonly SandboxProviderName[]; /** Mirrors the `image_builds.status` column. */ export const imageBuildStatusSchema = z.enum(["building", "ready", "failed", "superseded"]); diff --git a/packages/shared/src/types/integrations.ts b/packages/shared/src/types/integrations.ts index 5b82121d95..81a1e98430 100644 --- a/packages/shared/src/types/integrations.ts +++ b/packages/shared/src/types/integrations.ts @@ -282,6 +282,7 @@ export type SandboxSettings = z.infer; /** Every supported sandbox backend. Keep provider policy exhaustive over this union. */ export const SANDBOX_PROVIDER_NAMES = [ "modal", + "modal-vm", "daytona", "vercel", "opencomputer", @@ -293,6 +294,7 @@ export type SandboxProviderName = (typeof SANDBOX_PROVIDER_NAMES)[number]; const DEFAULT_SANDBOX_SETTING_CAPABILITIES = { resources: true, timeout: true }; const SANDBOX_SETTING_CAPABILITIES = { modal: DEFAULT_SANDBOX_SETTING_CAPABILITIES, + "modal-vm": DEFAULT_SANDBOX_SETTING_CAPABILITIES, daytona: { resources: false, timeout: false }, vercel: DEFAULT_SANDBOX_SETTING_CAPABILITIES, opencomputer: { resources: false, timeout: true }, diff --git a/packages/shared/src/types/sandbox-shutdown.test.ts b/packages/shared/src/types/sandbox-shutdown.test.ts index a6c4d42a63..c959fd9dcf 100644 --- a/packages/shared/src/types/sandbox-shutdown.test.ts +++ b/packages/shared/src/types/sandbox-shutdown.test.ts @@ -1,5 +1,10 @@ import { describe, expect, it } from "vitest"; -import { sandboxShutdownSchema } from "./sandbox-shutdown"; +import { z } from "zod"; +import { + sandboxPromptBlockReason, + sandboxShutdownSchema, + type SandboxShutdownState, +} from "./sandbox-shutdown"; describe("sandboxShutdownSchema", () => { it("round-trips authoritative recovery actions while remaining rolling-compatible", () => { @@ -15,4 +20,54 @@ describe("sandboxShutdownSchema", () => { sandboxShutdownSchema.safeParse({ ...base, availableRecoveryActions: ["resume"] }).success ).toBe(false); }); + + it("carries discard beside the action list so clients that predate it still parse the state", () => { + const projected = { + phase: "unknown", + expiresAtMs: null, + drainAtMs: null, + availableRecoveryActions: ["retry"], + discardAvailable: true, + }; + const predatingDiscard = sandboxShutdownSchema.omit({ discardAvailable: true }).extend({ + availableRecoveryActions: z.array(z.enum(["retry", "restore_saved"])).optional(), + }); + + expect(predatingDiscard.safeParse(projected).success).toBe(true); + expect(sandboxShutdownSchema.parse(projected)).toMatchObject({ + availableRecoveryActions: ["retry"], + discardAvailable: true, + }); + }); +}); + +const state = (phase: SandboxShutdownState["phase"]): SandboxShutdownState => ({ + phase, + expiresAtMs: null, + drainAtMs: null, +}); + +describe("sandboxPromptBlockReason", () => { + it.each(["failed", "unknown"] as const)( + "blocks prompts in %s without a recovery action", + (phase) => { + expect(sandboxPromptBlockReason(state(phase))).toContain("start a new session"); + } + ); + + it("points to recovery when one is available", () => { + expect( + sandboxPromptBlockReason({ ...state("failed"), availableRecoveryActions: ["retry"] }) + ).toContain("Use an available recovery action"); + expect(sandboxPromptBlockReason({ ...state("unknown"), discardAvailable: true })).toContain( + "Use an available recovery action" + ); + }); + + it.each(["running", "draining", "capturing", "saved", "restoring"] as const)( + "allows existing prompt queue behavior in %s", + (phase) => { + expect(sandboxPromptBlockReason(state(phase))).toBeNull(); + } + ); }); diff --git a/packages/shared/src/types/sandbox-shutdown.ts b/packages/shared/src/types/sandbox-shutdown.ts index 845cb11001..a1968a7ac2 100644 --- a/packages/shared/src/types/sandbox-shutdown.ts +++ b/packages/shared/src/types/sandbox-shutdown.ts @@ -1,6 +1,10 @@ import { z } from "zod"; -export const shutdownRecoveryActionSchema = z.enum(["retry", "restore_saved"]); +/** + * `retry` captures the held sandbox again, `restore_saved` continues from the last saved state, + * and `discard` stops the held sandbox and starts the next one from the repository. + */ +export const shutdownRecoveryActionSchema = z.enum(["retry", "restore_saved", "discard"]); export type ShutdownRecoveryAction = z.infer; /** Durable user-visible outcome, also included in reconnect snapshots. */ @@ -24,8 +28,23 @@ export const sandboxShutdownSchema = z.object({ hasRecoveryPoint: z.boolean().optional(), /** Server-authoritative actions currently safe for this exact sandbox generation. */ availableRecoveryActions: z.array(shutdownRecoveryActionSchema).optional(), + /** + * Whether `discard` is currently safe. Kept out of `availableRecoveryActions` + * so clients whose schema predates the action still parse the state. + */ + discardAvailable: z.boolean().optional(), /** Queued work requires an explicit user resume after an active prompt was interrupted. */ continuationPaused: z.boolean().optional(), }); export type SandboxShutdownState = z.infer; + +/** Failed/unknown shutdowns hold work until an explicit recovery succeeds. */ +export function sandboxPromptBlockReason( + state: SandboxShutdownState | null | undefined +): string | null { + if (state?.phase !== "failed" && state?.phase !== "unknown") return null; + return state.availableRecoveryActions?.length || state.discardAvailable + ? "New prompts are blocked until the sandbox is recovered. Use an available recovery action to continue." + : "New prompts are blocked. No recovery action is currently available; start a new session to continue. Unsaved changes may be missing."; +} diff --git a/packages/web/src/app/(app)/(sidebar)/session/[id]/page.tsx b/packages/web/src/app/(app)/(sidebar)/session/[id]/page.tsx index ca107807a1..a6e9574582 100644 --- a/packages/web/src/app/(app)/(sidebar)/session/[id]/page.tsx +++ b/packages/web/src/app/(app)/(sidebar)/session/[id]/page.tsx @@ -66,6 +66,7 @@ import { useSessionRename } from "@/hooks/use-session-rename"; import { useCurrentUserAuthorization } from "@/hooks/use-current-user-authorization"; import { resolveSessionCapabilities } from "@/lib/session-capabilities"; import { SandboxShutdownBanner } from "@/components/sandbox-shutdown-banner"; +import { sandboxPromptBlockReason } from "@open-inspect/shared/types/sandbox-shutdown"; type SessionState = ReturnType["sessionState"]; @@ -128,6 +129,7 @@ export default function SessionPage() { }); // Fixed at create; per-message model overrides must stay within it. const sessionHarness = sessionState?.harness ?? initialSnapshot.session.harness; + const sandboxBlockReason = sandboxPromptBlockReason(sessionState?.sandboxPreservation); const { selectedModel, reasoningEffort, @@ -156,7 +158,7 @@ export default function SessionPage() { reasoningEffort, loadingEnabledModels, sessionState?.status ?? DEFAULT_SESSION_STATUS, - ready && capabilities.collaborate && !sessionState?.budgetExhausted, + ready && capabilities.collaborate && !sessionState?.budgetExhausted && !sandboxBlockReason, shortcuts["send-prompt"] ); const [cancellingPromptIds, setCancellingPromptIds] = useState>(new Set()); @@ -378,15 +380,18 @@ export default function SessionPage() { draftLocked: isSubmitting || sessionAttachments.isUploading, sendBlocked: !ready || + Boolean(sandboxBlockReason) || Boolean(sessionState?.budgetExhausted) || modelAvailability.status === "unavailable", - blockedReason: sessionState?.budgetExhausted - ? canManageBudget - ? `Session cost limit reached at ${formatSessionCost(sessionState.totalCost ?? 0)} of ${formatSessionCost(sessionState.maxSessionCostUsd ?? 0)}. Raise or remove the limit to continue.` - : `Session cost limit reached at ${formatSessionCost(sessionState.totalCost ?? 0)} of ${formatSessionCost(sessionState.maxSessionCostUsd ?? 0)}. The session owner must raise or remove the limit to continue.` - : modelAvailability.status === "unavailable" - ? modelAvailability.message - : undefined, + blockedReason: + sandboxBlockReason ?? + (sessionState?.budgetExhausted + ? canManageBudget + ? `Session cost limit reached at ${formatSessionCost(sessionState.totalCost ?? 0)} of ${formatSessionCost(sessionState.maxSessionCostUsd ?? 0)}. Raise or remove the limit to continue.` + : `Session cost limit reached at ${formatSessionCost(sessionState.totalCost ?? 0)} of ${formatSessionCost(sessionState.maxSessionCostUsd ?? 0)}. The session owner must raise or remove the limit to continue.` + : modelAvailability.status === "unavailable" + ? modelAvailability.message + : undefined), submitError, inputRef, onSubmit: handleSubmit, diff --git a/packages/web/src/components/sandbox-shutdown-banner.test.tsx b/packages/web/src/components/sandbox-shutdown-banner.test.tsx index 92e57848f9..195bdd4a3c 100644 --- a/packages/web/src/components/sandbox-shutdown-banner.test.tsx +++ b/packages/web/src/components/sandbox-shutdown-banner.test.tsx @@ -219,11 +219,9 @@ describe("SandboxShutdownBanner", () => { onRecover={onRecover} /> ); - fireEvent.click(screen.getByRole("button", { name: "Retry shutdown" })); + fireEvent.click(screen.getByRole("button", { name: "Retry save" })); expect(onRecover).toHaveBeenCalledWith("retry"); - await waitFor(() => - expect(screen.getByRole("button", { name: "Retry shutdown" })).toBeEnabled() - ); + await waitFor(() => expect(screen.getByRole("button", { name: "Retry save" })).toBeEnabled()); fireEvent.click(screen.getByRole("button", { name: "Restore saved state" })); expect(confirm).toHaveBeenCalledWith( "Restore the last saved sandbox state? Changes since that save may be lost." @@ -234,6 +232,62 @@ describe("SandboxShutdownBanner", () => { expect(onRecover).toHaveBeenCalledWith("restore_saved"); }); + it("says unsaved changes may be lost when no save exists, and offers retry and discard", async () => { + const onRecover = acceptedRecovery(); + const confirm = vi.spyOn(window, "confirm").mockReturnValue(false); + render( + + ); + + expect(screen.getByRole("alert")).toHaveTextContent("The sandbox save could not be confirmed."); + expect(screen.getByRole("alert")).toHaveTextContent("Unsaved changes may be lost."); + expect(screen.getByRole("alert")).not.toHaveTextContent("since the last save"); + expect(screen.queryByRole("button", { name: "Restore saved state" })).not.toBeInTheDocument(); + + fireEvent.click(screen.getByRole("button", { name: "Discard and start fresh" })); + expect(confirm).toHaveBeenCalledWith(expect.stringContaining("unsaved changes will be lost")); + expect(onRecover).not.toHaveBeenCalled(); + confirm.mockReturnValue(true); + fireEvent.click(screen.getByRole("button", { name: "Discard and start fresh" })); + expect(onRecover).toHaveBeenCalledWith("discard"); + await waitFor(() => expect(screen.getByRole("button", { name: "Retry save" })).toBeEnabled()); + + fireEvent.click(screen.getByRole("button", { name: "Retry save" })); + expect(onRecover).toHaveBeenCalledWith("retry"); + }); + + it("warns about changes since the last save when a recovery point exists", () => { + render( + + ); + + expect(screen.getByRole("alert")).toHaveTextContent("The sandbox could not be saved."); + expect(screen.getByRole("alert")).toHaveTextContent("Changes since the last save may be lost."); + expect(screen.queryByRole("button", { name: "Retry save" })).not.toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Restore saved state" })).toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Discard and start fresh" })).toBeInTheDocument(); + }); + it.each([undefined, [] as ShutdownRecoveryAction[]])( "fails closed when projected recovery actions are %s even if a receipt exists", (availableRecoveryActions) => { @@ -250,8 +304,12 @@ describe("SandboxShutdownBanner", () => { /> ); - expect(screen.queryByRole("button", { name: "Retry shutdown" })).not.toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Retry save" })).not.toBeInTheDocument(); expect(screen.queryByRole("button", { name: "Restore saved state" })).not.toBeInTheDocument(); + expect( + screen.queryByRole("button", { name: "Discard and start fresh" }) + ).not.toBeInTheDocument(); + expect(screen.getByRole("alert")).toHaveTextContent("start a new session"); } ); @@ -275,10 +333,10 @@ describe("SandboxShutdownBanner", () => { /> ); - fireEvent.click(screen.getByRole("button", { name: "Retry shutdown" })); - expect(screen.getByRole("button", { name: "Retrying shutdown…" })).toBeDisabled(); + fireEvent.click(screen.getByRole("button", { name: "Retry save" })); + expect(screen.getByRole("button", { name: "Retrying save…" })).toBeDisabled(); expect(screen.getByRole("button", { name: "Restore saved state" })).toBeDisabled(); - fireEvent.click(screen.getByRole("button", { name: "Retrying shutdown…" })); + fireEvent.click(screen.getByRole("button", { name: "Retrying save…" })); expect(onRecover).toHaveBeenCalledOnce(); settle({ ok: false, reason: "timeout" }); @@ -306,10 +364,8 @@ describe("SandboxShutdownBanner", () => { /> ); - fireEvent.click(screen.getByRole("button", { name: "Retry shutdown" })); - await waitFor(() => - expect(screen.getByRole("button", { name: "Retry shutdown" })).toBeEnabled() - ); + fireEvent.click(screen.getByRole("button", { name: "Retry save" })); + await waitFor(() => expect(screen.getByRole("button", { name: "Retry save" })).toBeEnabled()); expect(screen.getByText(/Recovery was not confirmed/)).toBeInTheDocument(); }); @@ -330,7 +386,7 @@ describe("SandboxShutdownBanner", () => { /> ); - fireEvent.click(screen.getByRole("button", { name: "Retry shutdown" })); + fireEvent.click(screen.getByRole("button", { name: "Retry save" })); expect(await screen.findByText("Recovery is no longer eligible")).toBeInTheDocument(); expect(screen.queryByText(/Recovery was not confirmed/)).not.toBeInTheDocument(); }); diff --git a/packages/web/src/components/sandbox-shutdown-banner.tsx b/packages/web/src/components/sandbox-shutdown-banner.tsx index 4b74e3750a..eedb163f5e 100644 --- a/packages/web/src/components/sandbox-shutdown-banner.tsx +++ b/packages/web/src/components/sandbox-shutdown-banner.tsx @@ -6,6 +6,7 @@ import type { ShutdownRecoveryAction, } from "@open-inspect/shared/types/sandbox-shutdown"; import { cn } from "@/lib/utils"; +import { sandboxPromptBlockReason } from "@open-inspect/shared/types/sandbox-shutdown"; import type { ShutdownRecoveryResult } from "@/hooks/use-session-socket"; const PHASE_MESSAGES: Record< @@ -13,10 +14,13 @@ const PHASE_MESSAGES: Record< string > = { saved: "Sandbox saved and stopped.", - failed: "Final sandbox save failed. Changes since the last verified save may be missing.", - unknown: "Final sandbox save could not be confirmed. Changes may be missing.", + failed: "The sandbox could not be saved.", + unknown: "The sandbox save could not be confirmed.", }; +const DISCARD_CONFIRMATION = + "Discard this sandbox and start a new one from the repository? Its unsaved changes will be lost. Queued messages will run in the new sandbox."; + interface SandboxShutdownBannerProps { shutdown: SandboxShutdownState | null | undefined; onRecover?: (action: ShutdownRecoveryAction) => Promise; @@ -39,8 +43,12 @@ export function SandboxShutdownBanner({ shutdown, onRecover }: SandboxShutdownBa const recoveryActions = shutdown.availableRecoveryActions ?? []; const canRetry = recoveryActions.includes("retry"); const canRestoreSaved = recoveryActions.includes("restore_saved"); + const canDiscard = shutdown.discardAvailable === true; const canResumeQueuedWork = isContinuationPaused && canRestoreSaved; const detail = isError ? (shutdown.error ?? shutdown.reason) : undefined; + const lossWarning = shutdown.hasRecoveryPoint + ? "Changes since the last save may be lost." + : "Unsaved changes may be lost."; const recover = async (action: ShutdownRecoveryAction) => { if (!onRecover || pendingAction) return; @@ -74,6 +82,7 @@ export function SandboxShutdownBanner({ shutdown, onRecover }: SandboxShutdownBa )} > {PHASE_MESSAGES[phase]} + {isError && {lossWarning}} {isContinuationPaused && ( The sandbox was interrupted. Partial state was saved. Queued work will wait until you @@ -81,14 +90,17 @@ export function SandboxShutdownBanner({ shutdown, onRecover }: SandboxShutdownBa )} {detail && {detail}} - {phase === "failed" && canRetry && onRecover && ( + {isError && recoveryActions.length === 0 && !canDiscard && ( + {sandboxPromptBlockReason(shutdown)} + )} + {isError && canRetry && onRecover && ( )} {isError && canRestoreSaved && onRecover && ( @@ -109,6 +121,18 @@ export function SandboxShutdownBanner({ shutdown, onRecover }: SandboxShutdownBa {pendingAction === "restore_saved" ? "Restoring saved state…" : "Restore saved state"} )} + {isError && canDiscard && onRecover && ( + + )} {canResumeQueuedWork && onRecover && (