diff --git a/.env.example b/.env.example index ebc9031ac2..73d7570525 100644 --- a/.env.example +++ b/.env.example @@ -265,6 +265,9 @@ EXECUTION_TIMEOUT_MS= # "enforce" (default) fails spawn and build on oversized secret payloads; # "warn" only logs. Cloudflare: not set by Terraform. SECRETS_CAP_ENFORCEMENT= +# Session team enforcement: off, shadow, or on. Unset defaults to shadow. +# Private visibility applies in every mode. Cloudflare: var.teams_enforcement. +TEAMS_ENFORCEMENT= # --------------------------------------------------------------------------- # Logging diff --git a/.github/workflows/ci-python.yml b/.github/workflows/ci-python.yml index 8b2593523d..bc143f943c 100644 --- a/.github/workflows/ci-python.yml +++ b/.github/workflows/ci-python.yml @@ -220,16 +220,13 @@ jobs: python-version: "3.12" cache: "pip" - - name: Setup frozen image lock checker + - name: Setup uv uses: astral-sh/setup-uv@v7 with: version: "0.9.7" - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -e ../sandbox-runtime - pip install -e ".[dev]" + run: uv sync --frozen --extra dev - name: Run tests - run: pytest tests/ -v + run: uv run --frozen --extra dev pytest tests/ -v diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ba1cbb10bc..6341aaaa9e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -93,6 +93,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v7 + with: + persist-credentials: false - name: Setup Node.js uses: actions/setup-node@v7 @@ -151,6 +153,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v7 + with: + persist-credentials: false - name: Setup Node.js uses: actions/setup-node@v7 @@ -174,6 +178,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v7 + with: + persist-credentials: false - name: Setup Node.js uses: actions/setup-node@v7 @@ -221,6 +227,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v7 + with: + persist-credentials: false - name: Setup Node.js uses: actions/setup-node@v7 @@ -254,6 +262,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v7 + with: + persist-credentials: false - name: Setup Node.js uses: actions/setup-node@v7 @@ -279,6 +289,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v7 + with: + persist-credentials: false - name: Setup Node.js uses: actions/setup-node@v7 @@ -324,6 +336,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v7 + with: + persist-credentials: false - name: Setup Node.js uses: actions/setup-node@v7 diff --git a/CHANGELOG.md b/CHANGELOG.md index 76d0d7a845..875c8d5b21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,10 +2,24 @@ New features, integrations, and notable improvements to Open-Inspect — newest first. +## September 29, 2026 + +### Added + +`TEAMS_ENFORCEMENT` controls active-user session item routes (`/sessions/:id` and its subpaths) +using the persisted session row (`off`, `shadow` by default, or `on`). On those routes, private +visibility applies in every mode; team visibility and the delete ownership rule apply when `on`. +Workspace-wide session lists, bulk export, and WebSocket authorization follow in subsequent changes. +No route can make a session private or team-owned before those changes land. + ## September 28, 2026 ### Added +**Claude Sonnet 5.5.** Adds `anthropic/claude-sonnet-5-5` to the model picker and integrations, with +adaptive thinking controls from low through max. Claude Agent SDK 0.2.161 bundles Claude Code +2.1.284, which supports the new model. + OpenCode sessions using a connected ChatGPT subscription now report estimated model costs through the existing session cost display and spending limit. These are API-price equivalents, not additional subscription charges or an OpenAI invoice; estimates remain zero if catalog pricing is diff --git a/README.md b/README.md index 8f869e25fd..55fb6e5827 100644 --- a/README.md +++ b/README.md @@ -209,19 +209,14 @@ await configureGitIdentity({ Choose the AI model that fits your task, with per-session reasoning effort controls: -| Provider | Models | -| ---------------- | ----------------------------------------------------------------------- | -| Anthropic | Claude Haiku 4.5, Sonnet 4.5/4.6/5, Opus 4.5/4.6/4.7/4.8/5, Fable 5/5.1 | -| OpenAI | GPT 5.4, GPT 5.5, 5.3 Codex, 5.3 Codex Spark | -| xAI / SuperGrok | Grok models (opt-in) | -| OpenCode Zen | Kimi K2.5/K2.6/K3, MiniMax M2.5, Qwen3.7 Max, GLM 5/5.1/5.2 (opt-in) | -| Z.AI Coding Plan | GLM 5.2/5.3 (opt-in) | +Anthropic and OpenAI models are enabled by default. xAI / SuperGrok, OpenCode Zen and Go, Z.AI +Coding Plan, and DeepSeek models are opt-in. See [Available Models](docs/AVAILABLE_MODELS.md) for +current model IDs, descriptions, and reasoning efforts. OpenAI models work with your existing ChatGPT subscription via OAuth — no separate API key needed. Anthropic models can run on the **Claude Agent** harness with a connected Claude subscription; see [Using the Claude Agent Harness](docs/CLAUDE_AGENT.md). Grok models work with an eligible SuperGrok subscription through control-plane-managed OAuth. See -**[docs/AVAILABLE_MODELS.md](docs/AVAILABLE_MODELS.md)** for the full model list and **[docs/OPENAI_MODELS.md](docs/OPENAI_MODELS.md)** or **[docs/GROK_MODELS.md](docs/GROK_MODELS.md)** for subscription setup instructions. diff --git a/docs/AVAILABLE_MODELS.md b/docs/AVAILABLE_MODELS.md index 33eba020f0..e7b3b135bd 100644 --- a/docs/AVAILABLE_MODELS.md +++ b/docs/AVAILABLE_MODELS.md @@ -35,7 +35,8 @@ Accounts) applies only on the Claude Agent harness; OpenCode sessions use `ANTHR | `anthropic/claude-haiku-4-5` | Claude Haiku 4.5 | Fast and efficient | high, max | max | | `anthropic/claude-sonnet-4-5` | Claude Sonnet 4.5 | Balanced performance | high, max | max | | `anthropic/claude-sonnet-4-6` | Claude Sonnet 4.6 | Balanced, fast coding | low, medium, high, max | high | -| `anthropic/claude-sonnet-5` | Claude Sonnet 5 | Latest Sonnet, adaptive thinking | low, medium, high, xhigh, max | high | +| `anthropic/claude-sonnet-5` | Claude Sonnet 5 | Balanced performance, adaptive thinking | low, medium, high, xhigh, max | high | +| `anthropic/claude-sonnet-5-5` | Claude Sonnet 5.5 | Latest Sonnet, fast and intelligent | low, medium, high, xhigh, max | high | | `anthropic/claude-opus-4-5` | Claude Opus 4.5 | Most capable | high, max | max | | `anthropic/claude-opus-4-6` | Claude Opus 4.6 | Most capable, adaptive thinking | low, medium, high, max | high | | `anthropic/claude-opus-4-7` | Claude Opus 4.7 | Most capable, adaptive thinking | low, medium, high, xhigh, max | high | @@ -54,9 +55,9 @@ OpenAI models support connected ChatGPT provider accounts or `OPENAI_API_KEY` mo | ---------------------- | ------------- | ---------------------------------------------- | ----------------------------------- | -------------- | | `openai/gpt-5.4` | GPT 5.4 | Flagship model | none, low, medium, high, xhigh | Not set | | `openai/gpt-5.5` | GPT 5.5 | Latest flagship model | none, low, medium, high, xhigh | Not set | -| `openai/gpt-5.6-sol` | GPT 5.6 Sol | Frontier model for complex professional work | none, low, medium, high, xhigh | Not set | -| `openai/gpt-5.6-terra` | GPT 5.6 Terra | Balanced, cost-efficient everyday work | none, low, medium, high, xhigh | Not set | -| `openai/gpt-5.6-luna` | GPT 5.6 Luna | Fast, cost-efficient high-volume workloads | none, low, medium, high, xhigh | Not set | +| `openai/gpt-5.6-sol` | GPT 5.6 Sol | Frontier model for complex professional work | none, low, medium, high, xhigh | medium | +| `openai/gpt-5.6-terra` | GPT 5.6 Terra | Balanced, cost-efficient everyday work | none, low, medium, high, xhigh | medium | +| `openai/gpt-5.6-luna` | GPT 5.6 Luna | Fast, cost-efficient high-volume workloads | none, low, medium, high, xhigh, max | medium | | `openai/gpt-6-astra` | GPT-6 Astra | Most capable model for complex, demanding work | low, medium, high, xhigh, max | medium | | `openai/gpt-6-sol` | GPT-6 Sol | Complex coding and agentic workflows | none, low, medium, high, xhigh, max | medium | | `openai/gpt-6-luna` | GPT-6 Luna | Efficient model for focused, high-volume tasks | none, low, medium, high, xhigh, max | medium | diff --git a/docs/CLAUDE_AGENT.md b/docs/CLAUDE_AGENT.md index 48202c57b5..f4e4bb364d 100644 --- a/docs/CLAUDE_AGENT.md +++ b/docs/CLAUDE_AGENT.md @@ -196,8 +196,6 @@ fix instead. guard, the harness ignores the result of any turn it did not submit. - **Follow-ups queue.** Both harnesses hold follow-up prompts until the running turn completes. - **Image.** The sandbox image pins `claude-agent-sdk`, whose wheel bundles the `claude` binary. The - runtime manifest names the generation carrying the current pin under `harnessMinimumGeneration`, - so a Claude session never boots a prebuilt image from before that generation; this floor does not - touch OpenCode sessions' images or snapshots, since the global compatibility floor did not move. - Raise this floor whenever the SDK pin moves for a model the catalog advertises, otherwise a - session can be handed an older image whose bundled `claude` does not know that model. + runtime manifest's `harnessMinimumGeneration` controls which prepared images new Claude sessions + can use. Older images and resumed snapshots may lack newer models until rebuilt; a model request + can fail on a sandbox whose bundled CLI does not support it. diff --git a/docs/MODAL_DOCKER.md b/docs/MODAL_DOCKER.md index 675f07447c..56af3182d7 100644 --- a/docs/MODAL_DOCKER.md +++ b/docs/MODAL_DOCKER.md @@ -69,6 +69,48 @@ its generation is older than the materialization bound: the launch window plus t older launch that materializes later may briefly block a replacement, but the single allocation name and fenced credentials prevent overlapping work. +The authenticated `POST /api-resolve-vm-sandbox` endpoint is lookup-only. Its body contains exactly +`{"session_id":"...","sandbox_id":"..."}`; it accepts no launch settings or secrets. It finds the +running allocation by session name, checks the generation's ownership tags, and returns: + +```json +{ + "success": true, + "data": { + "sandbox_id": "generation-id", + "modal_object_id": "sb-real-id", + "code_server_url": null, + "code_server_password": null, + "vnc_url": null, + "vnc_password": null, + "ttyd_url": null, + "tunnel_urls": null, + "sandbox_backend": "modal-vm" + } +} +``` + +New VM allocations record versioned service flags and effective ports in provider-owned launch tags. +Only services enabled by these tags return URLs/passwords; extra tunnels use port-to-URL mappings. +Legacy allocations without these tags (or with unknown/incomplete metadata) resolve only the real +`modal_object_id`, not access credentials or tunnels. Resolve never infers enabled services from +environment variables, which may have contained user secrets on older allocations. Such VMs need a +new launch to recover interactive access. Resolve neither creates nor retires an allocation or +writes tunnel configuration. A stopped VM is not discoverable by name. Resolve does not return a +terminal access token; the control plane mints that token only when it still holds the generation's +sandbox auth token in memory. + +Create, restore, and resolve report typed HTTP 409 error `detail` values: + +- `not_visible`: resolve found no named allocation. +- `other_generation`: the ownership tags do not match. +- `window_closed`: create/restore missed the launch deadline with no owned allocation. +- `race_pending`: create/restore cannot yet see the winner after `AlreadyExistsError`, or resolve + found a VM whose enabled tunnel URLs are not all visible yet. + +Unexpected provider errors remain 500. The pending-reference stop endpoint retains its separate +`pending_reference_not_visible` response. + ## Switching backends Changing `SANDBOX_PROVIDER` is an operator cutover, not session migration. Existing sessions and diff --git a/packages/control-plane/src/authorization/request-audit.ts b/packages/control-plane/src/authorization/request-audit.ts index d0ebecf7a7..edc4485356 100644 --- a/packages/control-plane/src/authorization/request-audit.ts +++ b/packages/control-plane/src/authorization/request-audit.ts @@ -29,6 +29,8 @@ export type RouteAuthorizationDecision = kind: "allowed"; admission: "user" | "service" | "sandbox"; auditAllowed: boolean; + shadowReason?: string; + shadowDenials?: readonly { sessionId: string; reason: string }[]; }) | (AuthorizationDecisionEvidence & { kind: "denied"; @@ -40,7 +42,7 @@ export type RouteAuthorizationDecision = export function shouldAuditAllowedDecision( decision: Extract ): boolean { - return decision.auditAllowed; + return decision.auditAllowed || !!decision.shadowReason || !!decision.shadowDenials?.length; } /** @@ -59,6 +61,7 @@ export async function auditRouteAuthorizationDecision(input: { path: string; response: Response; decision: RouteAuthorizationDecision; + teamId?: string | null; }): Promise { const principal = input.ctx.principal; if (!principal) return; @@ -77,6 +80,20 @@ export async function auditRouteAuthorizationDecision(input: { const action = allowed ? AUTHORIZATION_DECISION_ACTIONS.allowed : AUTHORIZATION_DECISION_ACTIONS.denied; + const shadowCode = + decision.kind === "allowed" + ? decision.shadowDenials?.length + ? "shadow_denied:batch" + : decision.shadowReason + ? `shadow_denied:${decision.shadowReason}` + : null + : null; + const teamId = + input.teamId !== undefined + ? input.teamId + : input.ctx.childSessionAdmission + ? input.ctx.childSessionAdmission.row.ownerTeamId + : (input.ctx.sessionAdmission?.row.ownerTeamId ?? null); const metadata = { schema: AUTHORIZATION_DECISION_METADATA_SCHEMA, httpMethod: input.method, @@ -87,11 +104,14 @@ export async function auditRouteAuthorizationDecision(input: { ? { effectivePermissions: decision.effectivePermissions } : {}), ...(requiredPermission ? { requiredPermission } : {}), - responseCode: decision.kind === "denied" ? decision.reasonCode : null, + responseCode: decision.kind === "denied" ? decision.reasonCode : shadowCode, responseReason: decision.kind === "denied" ? decision.reason : null, requestId: input.ctx.request_id, traceId: input.ctx.trace_id, ...(decision.kind === "allowed" ? { admission: decision.admission } : {}), + ...(decision.kind === "allowed" && decision.shadowDenials?.length + ? { shadowDenials: decision.shadowDenials } + : {}), ...(principal.kind === "service" && principal.actor ? { actor: { @@ -110,8 +130,8 @@ export async function auditRouteAuthorizationDecision(input: { `INSERT INTO authorization_audit_events (id, occurred_at, request_id, principal_kind, actor_user_id_snapshot, actor_service_snapshot, action, resource_type, resource_id, - reason_code, operation_result, metadata_json) - VALUES (?, ?, ?, ?, ?, ?, ?, 'http_route', ?, ?, ?, ?)` + reason_code, operation_result, metadata_json, team_id) + VALUES (?, ?, ?, ?, ?, ?, ?, 'http_route', ?, ?, ?, ?, ?)` ) .bind( crypto.randomUUID(), @@ -122,9 +142,10 @@ export async function auditRouteAuthorizationDecision(input: { principal.kind === "service" ? principal.service : null, action, input.path, - decision.kind === "allowed" ? "authorization_allowed" : decision.reasonCode, + decision.kind === "allowed" ? (shadowCode ?? "authorization_allowed") : decision.reasonCode, allowed ? "applied" : "denied", - JSON.stringify(metadata) + JSON.stringify(metadata), + teamId ) .run(); } catch (cause) { @@ -137,3 +158,34 @@ export async function auditRouteAuthorizationDecision(input: { }); } } + +export async function auditPrivateSessionBreakGlass( + ctx: RequestContext, + sessionId: string, + teamId: string | null +): Promise { + const principal = ctx.principal; + const actorUserId = ctx.authorization?.userId; + if (!principal || !actorUserId) throw new Error("Missing private session break-glass actor"); + await ctx.db + .prepare( + `INSERT INTO authorization_audit_events + (id, occurred_at, request_id, principal_kind, actor_user_id_snapshot, + actor_service_snapshot, action, resource_type, resource_id, team_id, + reason_code, operation_result, metadata_json) + VALUES (?, ?, ?, ?, ?, ?, 'session.private_break_glass', 'session', ?, ?, ?, 'applied', ?)` + ) + .bind( + crypto.randomUUID(), + Date.now(), + ctx.request_id, + principal.kind, + actorUserId, + principal.kind === "service" ? principal.service : null, + sessionId, + teamId, + "session.private_break_glass", + JSON.stringify({ before: {}, requested: {}, after: {} }) + ) + .run(); +} diff --git a/packages/control-plane/src/authorization/session-admission.ts b/packages/control-plane/src/authorization/session-admission.ts new file mode 100644 index 0000000000..022c20f4ff --- /dev/null +++ b/packages/control-plane/src/authorization/session-admission.ts @@ -0,0 +1,107 @@ +import { checkSessionAccess, type SessionAction, type SessionViewer } from "@open-inspect/shared"; +import type { PermissionId } from "@open-inspect/shared/rbac"; +import type { TeamRole } from "@open-inspect/shared/types/teams"; +import { SessionCollaboratorStore } from "../db/session-collaborators"; +import { SessionIndexStore } from "../db/session-index"; +import { TeamMembershipStore } from "../db/team-memberships"; +import type { RequestContext } from "../http/request-context"; +import type { Env } from "../types"; +import { auditPrivateSessionBreakGlass } from "./request-audit"; +import { + legacyPermissionForAction, + parseTeamsEnforcementMode, + type TeamsEnforcementMode, +} from "./teams-enforcement"; + +export function teamsEnforcementMode(ctx: RequestContext, env: Env): TeamsEnforcementMode { + return (ctx.teamsEnforcementMode ??= parseTeamsEnforcementMode(env.TEAMS_ENFORCEMENT)); +} + +export function viewerFromContext( + ctx: RequestContext, + memberships: ReadonlyMap +): SessionViewer { + const authorization = ctx.authorization; + if (!authorization) { + if (ctx.principal?.kind === "service" && !ctx.principal.actor) + return { kind: "service", teamId: null }; + throw new Error("Missing request authorization"); + } + return { + kind: "user", + userId: authorization.userId, + roleKey: authorization.role.key, + permissions: authorization.permissions, + suspended: authorization.suspendedAt !== null, + memberships, + }; +} + +export type SessionAdmissionOutcome = + | { kind: "not_found" } + | { kind: "action_denied"; reason: string } + | { kind: "allowed"; legacyPermission: PermissionId | null }; + +/** Resolve one D1 session; a null slot is used by body-ID batches, not item routes. */ +export async function evaluateSessionAdmission( + ctx: RequestContext, + env: Env, + sessionId: string, + action: SessionAction, + slot: "session" | "child" | null = "session" +): Promise { + const mode = teamsEnforcementMode(ctx, env); + const row = await new SessionIndexStore(ctx.db).get(sessionId); + if (!row) return { kind: "not_found" }; + + if (mode === "off" && row.visibility !== "private") { + return { kind: "allowed", legacyPermission: legacyPermissionForAction(action) }; + } + + const memberships = + mode === "off" || !ctx.authorization + ? new Map() + : (ctx.sessionMemberships ??= await new TeamMembershipStore(ctx.db).listForUser( + ctx.authorization.userId + )); + const viewer = viewerFromContext(ctx, memberships); + const accessRow = { + ...row, + ownerUserId: row.userId ?? null, + collaboratorIds: await new SessionCollaboratorStore(ctx.db).listUserIds(sessionId), + }; + if (slot === "session") ctx.sessionAdmission = { row: accessRow, viewer }; + if (slot === "child") ctx.childSessionAdmission = { row: accessRow, viewer }; + + const read = checkSessionAccess(viewer, accessRow, "read"); + if ( + !read.allowed && + (mode === "on" || (row.visibility === "private" && read.reason === "private")) + ) { + return { kind: "not_found" }; + } + if (read.allowed && read.audit === "session.private_break_glass") { + await auditPrivateSessionBreakGlass(ctx, sessionId, row.ownerTeamId); + } + + // The signed route grant authorizes actorless actions; the service resolver only checks visibility. + const decision = viewer.kind === "service" ? null : checkSessionAccess(viewer, accessRow, action); + if (mode === "on" && decision && !decision.allowed) { + return { kind: "action_denied", reason: decision.reason }; + } + if (mode === "shadow") { + const reason = !read.allowed + ? read.reason + : decision && !decision.allowed + ? decision.reason + : null; + if (reason) { + if (slot === null) (ctx.shadowBatchDenials ??= []).push({ sessionId, reason }); + else ctx.shadowSessionDenial ??= reason; + } + } + return { + kind: "allowed", + legacyPermission: mode === "on" ? null : legacyPermissionForAction(action), + }; +} diff --git a/packages/control-plane/src/authorization/teams-enforcement.test.ts b/packages/control-plane/src/authorization/teams-enforcement.test.ts new file mode 100644 index 0000000000..a9fd84ed59 --- /dev/null +++ b/packages/control-plane/src/authorization/teams-enforcement.test.ts @@ -0,0 +1,26 @@ +import { describe, expect, it } from "vitest"; +import { SESSION_ACTIONS } from "@open-inspect/shared"; +import { legacyPermissionForAction, parseTeamsEnforcementMode } from "./teams-enforcement"; + +describe("teams enforcement", () => { + it("defaults to shadow and accepts only the three modes", () => { + expect(parseTeamsEnforcementMode(undefined)).toBe("shadow"); + expect(parseTeamsEnforcementMode("off")).toBe("off"); + expect(parseTeamsEnforcementMode("shadow")).toBe("shadow"); + expect(parseTeamsEnforcementMode("on")).toBe("on"); + expect(() => parseTeamsEnforcementMode("enabled")).toThrow(); + }); + + it("maps every resolver action to its pre-enforcement permission", () => { + expect(SESSION_ACTIONS.map((action) => legacyPermissionForAction(action))).toEqual([ + "sessions.read", + "sessions.collaborate", + "sessions.lifecycle", + "sessions.delete", + "sessions.sandbox_access", + "sessions.lifecycle", + "sessions.lifecycle", + "sessions.lifecycle", + ]); + }); +}); diff --git a/packages/control-plane/src/authorization/teams-enforcement.ts b/packages/control-plane/src/authorization/teams-enforcement.ts new file mode 100644 index 0000000000..6edb6da94b --- /dev/null +++ b/packages/control-plane/src/authorization/teams-enforcement.ts @@ -0,0 +1,28 @@ +import type { PermissionId } from "@open-inspect/shared/rbac"; +import type { SessionAction } from "@open-inspect/shared"; + +export type TeamsEnforcementMode = "off" | "shadow" | "on"; + +export function parseTeamsEnforcementMode(value: string | undefined): TeamsEnforcementMode { + if (value === undefined || value === "") return "shadow"; + if (value === "off" || value === "shadow" || value === "on") return value; + throw new Error(`Invalid TEAMS_ENFORCEMENT: ${value}`); +} + +export function legacyPermissionForAction(action: SessionAction): PermissionId { + switch (action) { + case "read": + return "sessions.read"; + case "collaborate": + return "sessions.collaborate"; + case "delete": + return "sessions.delete"; + case "sandbox": + return "sessions.sandbox_access"; + case "lifecycle": + case "move": + case "changeVisibility": + case "manageCollaborators": + return "sessions.lifecycle"; + } +} diff --git a/packages/control-plane/src/db/session-collaborators.ts b/packages/control-plane/src/db/session-collaborators.ts new file mode 100644 index 0000000000..eb24384886 --- /dev/null +++ b/packages/control-plane/src/db/session-collaborators.ts @@ -0,0 +1,43 @@ +import { z } from "zod"; +import type { SqlDatabase } from "./sql-database"; + +const collaboratorSchema = z.object({ session_id: z.string(), user_id: z.string() }); + +export class SessionCollaboratorStore { + constructor(private readonly db: SqlDatabase) {} + + async listUserIds(sessionId: string): Promise { + const rows = await this.db + .prepare("SELECT session_id, user_id FROM session_collaborators WHERE session_id = ?") + .bind(sessionId) + .all(); + return rows.results.map((row) => collaboratorSchema.parse(row).user_id); + } + + async listForUser(userId: string): Promise { + const rows = await this.db + .prepare("SELECT session_id, user_id FROM session_collaborators WHERE user_id = ?") + .bind(userId) + .all(); + return rows.results.map((row) => collaboratorSchema.parse(row).session_id); + } + + async add(sessionId: string, userId: string, addedBy: string): Promise { + const result = await this.db + .prepare( + `INSERT INTO session_collaborators (session_id, user_id, added_by, created_at) + VALUES (?, ?, ?, ?) ON CONFLICT (session_id, user_id) DO NOTHING` + ) + .bind(sessionId, userId, addedBy, Date.now()) + .run(); + return result.meta.changes > 0; + } + + async remove(sessionId: string, userId: string): Promise { + const result = await this.db + .prepare("DELETE FROM session_collaborators WHERE session_id = ? AND user_id = ?") + .bind(sessionId, userId) + .run(); + return result.meta.changes > 0; + } +} diff --git a/packages/control-plane/src/http/request-context.ts b/packages/control-plane/src/http/request-context.ts index 4fc1c03cc8..8218c7bb62 100644 --- a/packages/control-plane/src/http/request-context.ts +++ b/packages/control-plane/src/http/request-context.ts @@ -1,12 +1,16 @@ import type { EffectiveAuthorization } from "@open-inspect/shared/rbac"; import type { TeamCapabilities } from "@open-inspect/shared/types/team-access"; import type { Team } from "@open-inspect/shared/types/teams"; +import type { TeamRole } from "@open-inspect/shared/types/teams"; +import type { SessionAccessRow, SessionViewer } from "@open-inspect/shared"; import type { AuthenticationContext, Principal } from "../auth/principal"; import type { AuthenticationRequestServices } from "../auth/request-services"; import type { UserAuthRuntime } from "../auth/user/runtime"; import type { AutomationRow } from "../db/automation-store"; +import type { SessionEntry } from "../db/session-index"; import type { RequestMetrics } from "../db/instrumented-sql-database"; import type { BackgroundTasks } from "../platform-ports"; +import type { TeamsEnforcementMode } from "../authorization/teams-enforcement"; /** Automation resource admitted for the current mutation. */ export interface AutomationRouteAdmission { @@ -27,4 +31,10 @@ export type RequestContext = AuthenticationRequestServices & { authorization?: EffectiveAuthorization; automationAdmission?: AutomationRouteAdmission; teamAdmission?: { team: Team; access: TeamCapabilities }; + sessionAdmission?: { row: SessionEntry & SessionAccessRow; viewer: SessionViewer }; + childSessionAdmission?: { row: SessionEntry & SessionAccessRow; viewer: SessionViewer }; + sessionMemberships?: ReadonlyMap; + teamsEnforcementMode?: TeamsEnforcementMode; + shadowSessionDenial?: string; + shadowBatchDenials?: { sessionId: string; reason: string }[]; }; diff --git a/packages/control-plane/src/node/config.test.ts b/packages/control-plane/src/node/config.test.ts index ab099d8291..1247b97482 100644 --- a/packages/control-plane/src/node/config.test.ts +++ b/packages/control-plane/src/node/config.test.ts @@ -26,6 +26,12 @@ describe("readEnvConfig", () => { expect("LOG_LEVEL" in config).toBe(false); }); + it("rejects invalid teams enforcement modes at startup", () => { + expect(() => readEnvConfig({ ...REQUIRED, TEAMS_ENFORCEMENT: "enabled" })).toThrow( + "Invalid TEAMS_ENFORCEMENT" + ); + }); + it("names every missing required variable at once", () => { expect(() => readEnvConfig({ DEPLOYMENT_NAME: "test", TOKEN_ENCRYPTION_KEY: "" })).toThrow( "Missing required configuration: GITHUB_BOT_USERNAME, TOKEN_ENCRYPTION_KEY, PROVIDER_ACCOUNTS_ENCRYPTION_KEY, REPO_SECRETS_ENCRYPTION_KEY" diff --git a/packages/control-plane/src/node/config.ts b/packages/control-plane/src/node/config.ts index e1847eeaa3..f5581783f3 100644 --- a/packages/control-plane/src/node/config.ts +++ b/packages/control-plane/src/node/config.ts @@ -17,6 +17,7 @@ import { dirname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import type { EnvConfig } from "../types"; +import { parseTeamsEnforcementMode } from "../authorization/teams-enforcement"; /** A source of configuration values, `process.env` in production. */ export type ConfigSource = Record; @@ -94,6 +95,7 @@ const ENV_CONFIG_KEYS = { SANDBOX_BOOT_TIMEOUT_MS: true, EXECUTION_TIMEOUT_MS: true, SECRETS_CAP_ENFORCEMENT: true, + TEAMS_ENFORCEMENT: true, LOG_LEVEL: true, } as const satisfies Record; @@ -158,6 +160,7 @@ export function readEnvConfig(source: ConfigSource): EnvConfig { if (missing.length > 0) { throw new Error(`Missing required configuration: ${missing.join(", ")}`); } + parseTeamsEnforcementMode(config.TEAMS_ENFORCEMENT); return config as EnvConfig; } diff --git a/packages/control-plane/src/router.policy.test.ts b/packages/control-plane/src/router.policy.test.ts index b5567b7056..b81e1ee9a7 100644 --- a/packages/control-plane/src/router.policy.test.ts +++ b/packages/control-plane/src/router.policy.test.ts @@ -31,10 +31,10 @@ describe("route policy table", () => { }); }); - it("gates a single-session export with sessions.export", () => { + it("gates a single-session export with session read and sessions.export", () => { expect(routeFor("GET", "/sessions/session-1/export")?.authorization).toMatchObject({ kind: "active-user", - allOf: [{ permission: "sessions.export" }], + allOf: [{ kind: "session", action: "read" }, { permission: "sessions.export" }], }); }); @@ -117,6 +117,30 @@ describe("route policy table", () => { } }); + it("requires session admission on every active-user session item route", () => { + const exceptions: string[] = []; + for (const route of routes) { + if (!route.path.startsWith("/sessions/:id") || route.authorization.kind !== "active-user") + continue; + const identity = `${route.method} ${route.path}`; + if (exceptions.includes(identity)) continue; + expect( + route.authorization.allOf.some((requirement) => requirement.kind === "session"), + identity + ).toBe(true); + if (route.path.includes(":childId")) { + expect( + route.authorization.allOf.some( + (requirement) => + requirement.kind === "session" && requirement.sessionIdParam === "childId" + ), + identity + ).toBe(true); + } + } + expect(exceptions).toEqual([]); + }); + it.each([ ["GET", "/repos", [{ service: "slack-bot" }, { service: "linear-bot" }]], ["GET", "/repos/acme/widgets/metadata", [{ service: "github-bot" }]], @@ -233,7 +257,7 @@ describe("route policy table", () => { }); expect(routeFor("POST", "/sessions/session-1/ws-token")?.authorization).toMatchObject({ kind: "active-user", - allOf: [{ kind: "permission", permission: "sessions.read" }], + allOf: [{ kind: "session", action: "read" }], }); expect(routeFor("POST", "/sessions/session-1/stop")?.authorization).toMatchObject({ service: { kind: "actor", actorlessGrants: [{ service: "linear-bot" }] }, @@ -245,14 +269,26 @@ describe("route policy table", () => { expect(routeFor("POST", "/sessions/parent/children")?.authorization).toMatchObject({ kind: "active-user", allOf: [ + { kind: "session", action: "collaborate" }, { kind: "permission", permission: "sessions.create" }, - { kind: "permission", permission: "sessions.collaborate" }, ], }); expect(routeFor("GET", "/sessions/parent/children/child")?.authorization).toMatchObject({ kind: "active-user", - allOf: [{ kind: "permission", permission: "sessions.read" }], + allOf: [ + { kind: "session", action: "read", sessionIdParam: "id" }, + { kind: "session", action: "read", sessionIdParam: "childId" }, + ], }); + expect(routeFor("POST", "/sessions/parent/children/child/cancel")?.authorization).toMatchObject( + { + kind: "active-user", + allOf: [ + { kind: "session", action: "read", sessionIdParam: "id" }, + { kind: "session", action: "lifecycle", sessionIdParam: "childId" }, + ], + } + ); expect(routeFor("POST", "/internal/github-event")?.authorization).toMatchObject({ kind: "service", services: ["github-bot"], diff --git a/packages/control-plane/src/router.scm-credentials.test.ts b/packages/control-plane/src/router.scm-credentials.test.ts index 6e211fe616..974cb11591 100644 --- a/packages/control-plane/src/router.scm-credentials.test.ts +++ b/packages/control-plane/src/router.scm-credentials.test.ts @@ -7,6 +7,7 @@ import { signedServiceRequest, TEST_BACKGROUND_TASK_CONTEXT, TEST_SERVICE_SECRETS, + TEST_SESSION_ROW, } from "./router.test-support"; function routeFor(method: string, path: string) { @@ -55,6 +56,13 @@ function createEnv(options?: { actorAuthorized?: boolean }) { }; return identityStatement; } + if (options?.actorAuthorized && sql.includes("SELECT * FROM sessions")) { + const sessionStatement = { + bind: vi.fn(() => sessionStatement), + first: vi.fn(async () => TEST_SESSION_ROW), + }; + return sessionStatement; + } if ( options?.actorAuthorized && sql.includes("FROM users u") && diff --git a/packages/control-plane/src/router.session-prompt.test.ts b/packages/control-plane/src/router.session-prompt.test.ts index 90781a62dd..b015aed3b1 100644 --- a/packages/control-plane/src/router.session-prompt.test.ts +++ b/packages/control-plane/src/router.session-prompt.test.ts @@ -9,6 +9,7 @@ import { signedServiceRequest, TEST_BACKGROUND_TASK_CONTEXT, TEST_SERVICE_SECRETS, + TEST_SESSION_ROW, } from "./router.test-support"; vi.mock("./db/user-store", () => ({ @@ -73,11 +74,27 @@ function createEnv(sessionFetch: (request: Request) => Promise): Recor })), run: vi.fn(async () => ({ meta: { changes: 0 } })), }; + const sessionStatement = { + ...statement, + bind: vi.fn(() => sessionStatement), + first: vi.fn(async () => TEST_SESSION_ROW), + }; + const membershipStatement = { + ...statement, + bind: vi.fn(() => membershipStatement), + all: vi.fn(async () => ({ results: [] })), + }; return { ...TEST_SERVICE_SECRETS, SCM_PROVIDER: "github", DB: { - prepare: vi.fn(() => statement), + prepare: vi.fn((sql: string) => + sql.includes("SELECT * FROM sessions") + ? sessionStatement + : sql.includes("FROM team_memberships") || sql.includes("FROM session_collaborators") + ? membershipStatement + : statement + ), batch: vi.fn(), exec: vi.fn(), dump: vi.fn(), diff --git a/packages/control-plane/src/router.spawn-child.test.ts b/packages/control-plane/src/router.spawn-child.test.ts index 7496c6b84f..d80e08680b 100644 --- a/packages/control-plane/src/router.spawn-child.test.ts +++ b/packages/control-plane/src/router.spawn-child.test.ts @@ -660,7 +660,8 @@ describe("handleSpawnChild prompt enqueue handling", () => { expect(response.status).toBe(400); await expect(response.json()).resolves.toEqual({ error: "title and prompt are required" }); - expect(SessionIndexStore).not.toHaveBeenCalled(); + expect(store.get).toHaveBeenCalledWith(parentId); + expect(store.create).not.toHaveBeenCalled(); }); it("returns 400 for a child spawn body that is not JSON", async () => { @@ -689,7 +690,8 @@ describe("handleSpawnChild prompt enqueue handling", () => { expect(response.status).toBe(400); await expect(response.json()).resolves.toEqual({ error: "Invalid JSON body" }); - expect(SessionIndexStore).not.toHaveBeenCalled(); + expect(store.get).toHaveBeenCalledWith(parentId); + expect(store.create).not.toHaveBeenCalled(); }); it("returns 500 for a malformed parent spawn context", async () => { diff --git a/packages/control-plane/src/router.test-support.ts b/packages/control-plane/src/router.test-support.ts index 430680f35a..02fcc3cd92 100644 --- a/packages/control-plane/src/router.test-support.ts +++ b/packages/control-plane/src/router.test-support.ts @@ -12,6 +12,7 @@ import { createTestBackgroundTasks } from "./background-tasks.test-support"; import { BUILT_IN_ROLE_REGISTRY, type PermissionId } from "@open-inspect/shared/rbac"; import type { CacheStore } from "@open-inspect/shared/cache-store"; import type { SqlDatabase, SqlStatement } from "./db/sql-database"; +import type { SessionRow } from "./db/session-row"; import type { SessionRuntimeDispatch } from "./session/runtime-client"; import { cloudflareHost } from "./cloudflare/http-host"; import { createControlPlaneApp, type RouteModule } from "./routing/hono-app"; @@ -92,6 +93,52 @@ export function emptyStatement(): SqlStatement { return statement; } +/** A persisted workspace session for route tests that are about handler behavior. */ +export const TEST_SESSION_ROW: SessionRow = { + id: "session-1", + title: null, + repo_owner: null, + repo_name: null, + harness: "opencode", + model: "test/model", + reasoning_effort: null, + base_branch: null, + status: "completed", + parent_session_id: null, + root_session_id: null, + spawn_source: "user", + spawn_depth: 0, + automation_id: null, + automation_run_id: null, + scm_login: null, + user_id: TEST_USER_ID, + owner_team_id: null, + visibility: "workspace", + project_id: null, + total_cost: 0, + active_duration_ms: 0, + message_count: 0, + pr_count: 0, + input_tokens: 0, + output_tokens: 0, + reasoning_tokens: 0, + cache_read_tokens: 0, + cache_write_tokens: 0, + environment_id: null, + created_at: 1, + updated_at: 1, +}; + +export function sessionAdmissionStatement(sql: string): SqlStatement | null { + if (!sql.includes("SELECT * FROM sessions")) return null; + const statement: SqlStatement = { + ...emptyStatement(), + bind: () => statement, + first: async () => TEST_SESSION_ROW as T, + }; + return statement; +} + /** * A database whose effective-authorization lookup answers for one active * user, for request-level unit tests of admitted handlers. The two diff --git a/packages/control-plane/src/routes/session-attachments.ts b/packages/control-plane/src/routes/session-attachments.ts index 6032bf282f..76f08effde 100644 --- a/packages/control-plane/src/routes/session-attachments.ts +++ b/packages/control-plane/src/routes/session-attachments.ts @@ -52,7 +52,7 @@ import { GITHUB_SANDBOX_FALLBACK_ROUTE, GITHUB_USER_OR_SERVICE_ROUTE, json, - requirePermission, + requireSession, } from "./shared"; import { type SessionRouteContext, dispatchSession } from "./session-route"; @@ -243,13 +243,13 @@ sessionAttachmentRoutes.post( "/sessions/:id/attachments", admit({ ...GITHUB_USER_OR_SERVICE_ROUTE, - authorization: requirePermission("sessions.collaborate"), + authorization: requireSession("collaborate"), }), (c) => dispatchSession(c, handleAttachmentPost) ); sessionAttachmentRoutes.get( "/sessions/:id/attachments/:attachmentId", - admit({ ...GITHUB_SANDBOX_FALLBACK_ROUTE, authorization: requirePermission("sessions.read") }), + admit({ ...GITHUB_SANDBOX_FALLBACK_ROUTE, authorization: requireSession("read") }), (c) => dispatchSession(c, handleAttachmentGet) ); diff --git a/packages/control-plane/src/routes/session-batch-archive.ts b/packages/control-plane/src/routes/session-batch-archive.ts index db6769d887..b69c71c0a3 100644 --- a/packages/control-plane/src/routes/session-batch-archive.ts +++ b/packages/control-plane/src/routes/session-batch-archive.ts @@ -7,6 +7,7 @@ import { createLogger } from "../logger"; import { admit } from "../routing/admit"; import type { ControlPlaneHonoEnv } from "../routing/hono-env"; import { archiveSessionBatch } from "../session/batch-archive"; +import { evaluateSessionAdmission, teamsEnforcementMode } from "../authorization/session-admission"; import { parseBody } from "./body"; import type { SessionRuntimeClient } from "../session/runtime-client"; import { dispatchSession } from "./session-route"; @@ -31,7 +32,7 @@ sessionBatchArchiveRoutes.post( c, async ( request, - _env, + env, _params, ctx: UserRouteContext & { sessionRuntime: SessionRuntimeClient } ) => { @@ -41,13 +42,34 @@ sessionBatchArchiveRoutes.post( trace_id: ctx.trace_id, request_id: ctx.request_id, }); - const results = await archiveSessionBatch(body.sessionIds, ctx.sessionRuntime, log); + try { + teamsEnforcementMode(ctx, env); + } catch { + return json( + { error: "Authorization unavailable", code: "authorization_unavailable" }, + 503 + ); + } + const eligible: string[] = []; + const skipped: SessionBatchArchiveResponse["skipped"] = []; + for (const sessionId of body.sessionIds) { + const admission = await evaluateSessionAdmission(ctx, env, sessionId, "lifecycle", null); + if (admission.kind !== "allowed") { + skipped.push({ + sessionId, + reason: admission.kind === "not_found" ? "not_found" : "missing_permission", + }); + continue; + } + eligible.push(sessionId); + } + const results = await archiveSessionBatch(eligible, ctx.sessionRuntime, log); log.info("Session batch archive completed", { event: "session.batch_archive", user_id: ctx.principal.userId, results, }); - return json({ results } satisfies SessionBatchArchiveResponse); + return json({ results, skipped } satisfies SessionBatchArchiveResponse); } ) ); diff --git a/packages/control-plane/src/routes/session-child-spawn.ts b/packages/control-plane/src/routes/session-child-spawn.ts index 06c591619b..ea35d37d72 100644 --- a/packages/control-plane/src/routes/session-child-spawn.ts +++ b/packages/control-plane/src/routes/session-child-spawn.ts @@ -37,6 +37,7 @@ import { GITHUB_SANDBOX_FALLBACK_ROUTE, json, permissionRequirement, + sessionRequirement, requireAll, } from "./shared"; import { type SessionRouteContext, dispatchSession } from "./session-route"; @@ -380,8 +381,8 @@ sessionChildSpawnRoutes.post( admit({ ...GITHUB_SANDBOX_FALLBACK_ROUTE, authorization: requireAll( - permissionRequirement("sessions.create"), - permissionRequirement("sessions.collaborate") + sessionRequirement("collaborate"), + permissionRequirement("sessions.create") ), }), (c) => dispatchSession(c, handleSpawnChild) diff --git a/packages/control-plane/src/routes/session-children.test.ts b/packages/control-plane/src/routes/session-children.test.ts index be5c344599..8038d73c47 100644 --- a/packages/control-plane/src/routes/session-children.test.ts +++ b/packages/control-plane/src/routes/session-children.test.ts @@ -1,4 +1,6 @@ import { afterEach, describe, expect, it, vi } from "vitest"; +import { evaluateSessionAdmission } from "../authorization/session-admission"; +import type * as SessionAdmissionModule from "../authorization/session-admission"; import { SessionIndexStore } from "../db/session-index"; import { resolveSandboxSettings } from "../session/integration-settings-resolution"; import type { SessionRuntimeClient } from "../session/runtime-client"; @@ -13,6 +15,11 @@ vi.mock("../session/integration-settings-resolution", () => ({ resolveSandboxSettings: vi.fn(), })); +vi.mock("../authorization/session-admission", async (importOriginal) => ({ + ...(await importOriginal()), + evaluateSessionAdmission: vi.fn(), +})); + function routeMatch(path: string, pattern: string): { id: string; childId: string } { const match = path.match(routePathPattern(pattern)); if (!match?.groups?.id || !match.groups.childId) throw new Error("Expected route match"); @@ -53,6 +60,10 @@ describe("handleListChildren", () => { afterEach(() => vi.restoreAllMocks()); it("projects viewer-neutral child summaries through the shared schema", async () => { + vi.mocked(evaluateSessionAdmission).mockResolvedValue({ + kind: "allowed", + legacyPermission: "sessions.read", + }); vi.spyOn(SessionIndexStore.prototype, "listByParent").mockResolvedValue([ { id: "child", @@ -119,6 +130,13 @@ describe("handleListChildren", () => { }, ], }); + expect(evaluateSessionAdmission).toHaveBeenCalledWith( + expect.anything(), + expect.anything(), + "child", + "read", + null + ); }); }); diff --git a/packages/control-plane/src/routes/session-children.ts b/packages/control-plane/src/routes/session-children.ts index e74df8dd03..276332e416 100644 --- a/packages/control-plane/src/routes/session-children.ts +++ b/packages/control-plane/src/routes/session-children.ts @@ -11,6 +11,7 @@ import { import { DEFAULT_MAX_CONCURRENT_CHILD_SESSIONS } from "@open-inspect/shared/types/integrations"; import { childSessionListResponseSchema } from "@open-inspect/shared/types/sessions"; import { SessionIndexStore, type ChildAdmissionLease } from "../db/session-index"; +import { evaluateSessionAdmission } from "../authorization/session-admission"; import { createLogger } from "../logger"; import { SessionInternalPaths } from "../session/contracts"; import { resolveSandboxSettings } from "../session/integration-settings-resolution"; @@ -21,7 +22,9 @@ import { GITHUB_SANDBOX_FALLBACK_ROUTE, json, NO_AUTHORIZATION, - requirePermission, + requireAll, + requireSession, + sessionRequirement, SCM_AGNOSTIC_SANDBOX_ROUTE, type RequestContext, } from "./shared"; @@ -39,8 +42,17 @@ export async function handleListChildren( const sessionStore = new SessionIndexStore(ctx.db); const children = await sessionStore.listByParent(parentId); + const visible = []; + for (const child of children) { + if ( + ctx.principal?.kind === "sandbox" || + (await evaluateSessionAdmission(ctx, env, child.id, "read", null)).kind === "allowed" + ) { + visible.push(child); + } + } - return json(childSessionListResponseSchema.parse({ children })); + return json(childSessionListResponseSchema.parse({ children: visible })); } export async function handleGetChild( @@ -260,19 +272,25 @@ export const sessionChildRoutes = new Hono(); sessionChildRoutes.get( "/sessions/:id/children", - admit({ ...GITHUB_SANDBOX_FALLBACK_ROUTE, authorization: requirePermission("sessions.read") }), + admit({ ...GITHUB_SANDBOX_FALLBACK_ROUTE, authorization: requireSession("read") }), (c) => dispatch(c, handleListChildren) ); sessionChildRoutes.get( "/sessions/:id/children/:childId", - admit({ ...GITHUB_SANDBOX_FALLBACK_ROUTE, authorization: requirePermission("sessions.read") }), + admit({ + ...GITHUB_SANDBOX_FALLBACK_ROUTE, + authorization: requireAll(sessionRequirement("read"), sessionRequirement("read", "childId")), + }), (c) => dispatchSession(c, handleGetChild) ); sessionChildRoutes.post( "/sessions/:id/children/:childId/cancel", admit({ ...GITHUB_SANDBOX_FALLBACK_ROUTE, - authorization: requirePermission("sessions.lifecycle"), + authorization: requireAll( + sessionRequirement("read"), + sessionRequirement("lifecycle", "childId") + ), }), (c) => dispatchSession(c, handleCancelChild) ); diff --git a/packages/control-plane/src/routes/session-diffs.ts b/packages/control-plane/src/routes/session-diffs.ts index 86b2425e33..cff87e04e8 100644 --- a/packages/control-plane/src/routes/session-diffs.ts +++ b/packages/control-plane/src/routes/session-diffs.ts @@ -14,7 +14,7 @@ import { error, SCM_AGNOSTIC_SANDBOX_FALLBACK_ROUTE, SCM_AGNOSTIC_USER_OR_SERVICE_ROUTE, - requirePermission, + requireSession, } from "./shared"; import { type SessionRouteContext, dispatchSession } from "./session-route"; import type { Env } from "../types"; @@ -172,11 +172,11 @@ export const sessionDiffRoutes = new Hono(); const DIFF_READ = admit({ ...SCM_AGNOSTIC_USER_OR_SERVICE_ROUTE, - authorization: requirePermission("sessions.read"), + authorization: requireSession("read"), }); const DIFF_WRITE = admit({ ...SCM_AGNOSTIC_SANDBOX_FALLBACK_ROUTE, - authorization: requirePermission("sessions.collaborate"), + authorization: requireSession("collaborate"), }); sessionDiffRoutes.get("/sessions/:id/diff", DIFF_READ, (c) => dispatchSession(c, handleDiffState)); @@ -193,7 +193,7 @@ sessionDiffRoutes.post( "/sessions/:id/diff/retry", admit({ ...SCM_AGNOSTIC_USER_OR_SERVICE_ROUTE, - authorization: requirePermission("sessions.lifecycle"), + authorization: requireSession("lifecycle"), }), (c) => dispatchSession(c, handleDiffRetry) ); diff --git a/packages/control-plane/src/routes/session-export.test.ts b/packages/control-plane/src/routes/session-export.test.ts index dc2e3595dd..2eb2208851 100644 --- a/packages/control-plane/src/routes/session-export.test.ts +++ b/packages/control-plane/src/routes/session-export.test.ts @@ -20,6 +20,8 @@ import { createTestRequestHandler, TEST_BACKGROUND_TASK_CONTEXT, TEST_SERVICE_SECRETS, + sessionAdmissionStatement, + emptyStatement, } from "../router.test-support"; import type { PermissionId } from "@open-inspect/shared/rbac"; import type { ListSessionsForExportOptions, SessionExportRow } from "../db/session-export-store"; @@ -76,7 +78,10 @@ vi.mock("../logger", () => ({ const USER_PRINCIPAL: Principal = { kind: "user", userId: "user-1" }; function createEnv(permissions?: readonly PermissionId[]): Env { - const db = permissions ? authorizationDatabase({ permissions }) : authorizationDatabase(); + const db = authorizationDatabase({ + permissions, + statement: (sql) => sessionAdmissionStatement(sql) ?? emptyStatement(), + }); return createTestEnv({ ...TEST_SERVICE_SECRETS, DB: db, @@ -114,7 +119,7 @@ describe("GET /sessions/:id/export", () => { mocks.runtimeFetch.mockReset(); }); - it("refuses viewers before looking up the session", async () => { + it("refuses viewers without sessions.export after session admission", async () => { const response = await callExport( {}, { sessionId: "session-1", permissions: ["sessions.read"] } diff --git a/packages/control-plane/src/routes/session-export.ts b/packages/control-plane/src/routes/session-export.ts index 1d46164963..a2564b4fef 100644 --- a/packages/control-plane/src/routes/session-export.ts +++ b/packages/control-plane/src/routes/session-export.ts @@ -52,7 +52,14 @@ import type { SessionRuntimeClient } from "../session/runtime-client"; import type { Env } from "../types"; import { parseQuery } from "./query"; import { dispatchSession, type SessionRouteContext } from "./session-route"; -import { error, SCM_AGNOSTIC_USER_OR_SERVICE_ROUTE, requirePermission } from "./shared"; +import { + error, + SCM_AGNOSTIC_USER_OR_SERVICE_ROUTE, + requirePermission, + requireAll, + permissionRequirement, + sessionRequirement, +} from "./shared"; export const EXPORT_SCHEMA_VERSION = TRACE_EXPORT_SCHEMA_VERSION; const MAX_EXPORT_LIMIT = 500; @@ -350,6 +357,12 @@ const EXPORT_READ = admit({ export const sessionExportRoutes = new Hono(); sessionExportRoutes.get("/sessions/export", EXPORT_READ, (c) => dispatchSession(c, handleExport)); -sessionExportRoutes.get("/sessions/:id/export", EXPORT_READ, (c) => - dispatchSession(c, handleSingleExport) +sessionExportRoutes.get( + "/sessions/:id/export", + admit({ + ...SCM_AGNOSTIC_USER_OR_SERVICE_ROUTE, + authorization: requireAll(sessionRequirement("read"), permissionRequirement("sessions.export")), + cacheControl: "private, no-store", + }), + (c) => dispatchSession(c, handleSingleExport) ); diff --git a/packages/control-plane/src/routes/session-index.ts b/packages/control-plane/src/routes/session-index.ts index d531b6b951..7a49764130 100644 --- a/packages/control-plane/src/routes/session-index.ts +++ b/packages/control-plane/src/routes/session-index.ts @@ -26,6 +26,7 @@ import { json, SCM_AGNOSTIC_HUMAN_USER_ROUTE, requirePermission, + requireSession, type RequestContext, type UserRouteContext, } from "./shared"; @@ -280,11 +281,11 @@ sessionIndexRoutes.get( ); sessionIndexRoutes.patch( "/sessions/:id/read-state", - admit({ ...SCM_AGNOSTIC_HUMAN_USER_ROUTE, authorization: requirePermission("sessions.read") }), + admit({ ...SCM_AGNOSTIC_HUMAN_USER_ROUTE, authorization: requireSession("read") }), (c) => dispatch(c, handlePatchReadState) ); sessionIndexRoutes.delete( "/sessions/:id", - admit({ ...GITHUB_USER_OR_SERVICE_ROUTE, authorization: requirePermission("sessions.delete") }), + admit({ ...GITHUB_USER_OR_SERVICE_ROUTE, authorization: requireSession("delete") }), (c) => dispatch(c, handleDeleteSession) ); diff --git a/packages/control-plane/src/routes/session-media-stream.ts b/packages/control-plane/src/routes/session-media-stream.ts index f9bf6a6a2e..78e1f97b43 100644 --- a/packages/control-plane/src/routes/session-media-stream.ts +++ b/packages/control-plane/src/routes/session-media-stream.ts @@ -13,7 +13,7 @@ import { createStoredObjectResponse, } from "./responses/stored-object-response"; import { getSessionArtifactFromRuntime } from "./session-media-artifacts"; -import { error, GITHUB_USER_OR_SERVICE_ROUTE, requirePermission } from "./shared"; +import { error, GITHUB_USER_OR_SERVICE_ROUTE, requireSession } from "./shared"; import { type SessionRouteContext, dispatchSession } from "./session-route"; const logger = createLogger("router:session-media"); @@ -144,7 +144,7 @@ sessionMediaStreamRoutes.get( "/sessions/:id/media/:artifactId", admit({ ...GITHUB_USER_OR_SERVICE_ROUTE, - authorization: requirePermission("sessions.read", { + authorization: requireSession("read", { actorlessGrants: [{ service: "slack-bot" }], }), }), diff --git a/packages/control-plane/src/routes/session-media-upload.ts b/packages/control-plane/src/routes/session-media-upload.ts index b17d8e3613..b1881c5710 100644 --- a/packages/control-plane/src/routes/session-media-upload.ts +++ b/packages/control-plane/src/routes/session-media-upload.ts @@ -24,7 +24,7 @@ import { import type { ObjectStorage } from "../storage/object-storage"; import type { Env } from "../types"; import { listSessionArtifactsFromRuntime, persistMediaArtifact } from "./session-media-artifacts"; -import { error, GITHUB_SANDBOX_FALLBACK_ROUTE, json, requirePermission } from "./shared"; +import { error, GITHUB_SANDBOX_FALLBACK_ROUTE, json, requireSession } from "./shared"; import { type SessionRouteContext, dispatchSession } from "./session-route"; function getRequiredFormString(value: MultipartFieldValue | null, name: string): string | Response { @@ -247,7 +247,7 @@ sessionMediaUploadRoutes.post( "/sessions/:id/media", admit({ ...GITHUB_SANDBOX_FALLBACK_ROUTE, - authorization: requirePermission("sessions.collaborate"), + authorization: requireSession("collaborate"), }), (c) => dispatchSession(c, handleMediaUpload) ); diff --git a/packages/control-plane/src/routes/session-prompt.ts b/packages/control-plane/src/routes/session-prompt.ts index 1331a5f44c..e8c8f618e7 100644 --- a/packages/control-plane/src/routes/session-prompt.ts +++ b/packages/control-plane/src/routes/session-prompt.ts @@ -28,7 +28,7 @@ import { type GitHubEnrichment, } from "../session/identity"; import type { Env } from "../types"; -import { error, json, GITHUB_USER_OR_SERVICE_ROUTE, requirePermission } from "./shared"; +import { error, json, GITHUB_USER_OR_SERVICE_ROUTE, requireSession } from "./shared"; import { parseJsonBody } from "./body"; import { type SessionRouteContext, dispatchSession } from "./session-route"; @@ -175,7 +175,7 @@ sessionPromptRoutes.post( "/sessions/:id/prompt", admit({ ...GITHUB_USER_OR_SERVICE_ROUTE, - authorization: requirePermission("sessions.collaborate"), + authorization: requireSession("collaborate"), }), (c) => dispatchSession(c, handleSessionPrompt) ); diff --git a/packages/control-plane/src/routes/session-pull-requests.ts b/packages/control-plane/src/routes/session-pull-requests.ts index b49594a598..c91cc0222e 100644 --- a/packages/control-plane/src/routes/session-pull-requests.ts +++ b/packages/control-plane/src/routes/session-pull-requests.ts @@ -3,7 +3,7 @@ import { admit } from "../routing/admit"; import type { ControlPlaneHonoEnv } from "../routing/hono-env"; import { SessionInternalPaths } from "../session/contracts"; import type { Env } from "../types"; -import { GITHUB_USER_OR_SERVICE_ROUTE, requirePermission } from "./shared"; +import { GITHUB_USER_OR_SERVICE_ROUTE, requireSession } from "./shared"; import { type SessionRouteContext, dispatchSession } from "./session-route"; /** @@ -31,7 +31,7 @@ sessionPullRequestRoutes.post( "/sessions/:id/pull-requests/refresh", admit({ ...GITHUB_USER_OR_SERVICE_ROUTE, - authorization: requirePermission("sessions.lifecycle"), + authorization: requireSession("lifecycle"), }), (c) => dispatchSession(c, handleRefreshPullRequests) ); diff --git a/packages/control-plane/src/routes/session-route.ts b/packages/control-plane/src/routes/session-route.ts index 1b21c063ef..7fc5c8fbeb 100644 --- a/packages/control-plane/src/routes/session-route.ts +++ b/packages/control-plane/src/routes/session-route.ts @@ -13,7 +13,7 @@ export function withSessionRuntime( env: Env, ctx: Context ): Context & { sessionRuntime: SessionRuntimeClient } { - return { ...ctx, sessionRuntime: createSessionRuntimeClient(env, ctx) }; + return Object.assign(ctx, { sessionRuntime: createSessionRuntimeClient(env, ctx) }); } /** Run a session handler for an admitted request, with the runtime client bound to it. */ diff --git a/packages/control-plane/src/routes/session-runtime-proxy.test.ts b/packages/control-plane/src/routes/session-runtime-proxy.test.ts index b2ce928f21..1ca6cdff4b 100644 --- a/packages/control-plane/src/routes/session-runtime-proxy.test.ts +++ b/packages/control-plane/src/routes/session-runtime-proxy.test.ts @@ -11,6 +11,7 @@ import { TEST_SERVICE_SECRETS, createTestRequestHandler, fakeSessionRuntimeDispatch, + TEST_SESSION_ROW, } from "../router.test-support"; import { SessionInternalPaths } from "../session/contracts"; import type { Env } from "../types"; @@ -34,6 +35,7 @@ const SANDBOX_TOKEN_HEADERS = { Authorization: "Bearer sandbox-token" }; type DatabaseOptions = { /** Custom-role grants for user-1; omitted means the owner role with every permission. */ permissions?: PermissionId[]; + visibility?: "private"; /** Answers every statement admission and the proxy's own reads do not own. */ delegate?: SqlDatabase; }; @@ -50,9 +52,15 @@ function createDatabase(options: DatabaseOptions = {}): SqlDatabase { if (sql.includes("FROM role_permissions")) { return (options.permissions ?? []).map((permission_id) => ({ permission_id })); } + if (sql.includes("FROM team_memberships") || sql.includes("FROM session_collaborators")) + return []; return null; }; const row = (sql: string): unknown => { + if (sql.includes("SELECT * FROM sessions")) + return options.visibility === "private" + ? { ...TEST_SESSION_ROW, visibility: "private", user_id: "another-user" } + : TEST_SESSION_ROW; if (sql.includes("FROM users u")) return { user_id: "user-1", suspended_at: null, ...role }; if (sql.includes("FROM session_model_provider_auth")) { return { @@ -205,54 +213,73 @@ describe("session runtime proxy routes", () => { }); it.each([ - { permissions: ["sessions.read"] as PermissionId[], exposed: false }, + { + permissions: ["sessions.read"] as PermissionId[], + exposed: false, + mode: "shadow", + visibility: "workspace", + }, { permissions: ["sessions.read", "sessions.sandbox_access"] as PermissionId[], exposed: true, + mode: "shadow", + visibility: "workspace", }, - ])("scopes snapshot sandbox locations to sandbox access ($exposed)", async (input) => { - const fetch = vi.fn(async () => - Response.json({ - session: { - id: "session-1", - title: "Session", - repoOwner: "acme", - repoName: "web", - baseBranch: "main", - branchName: "feature", - status: "active", - sandboxStatus: "ready", - messageCount: 0, - createdAt: 1, - codeServerUrl: "https://code.example", - vncUrl: "https://vnc.example", - ttydUrl: "https://terminal.example", - tunnelUrls: { "3000": "https://app.example" }, - sandboxDashboardUrl: "https://provider.example", - }, - artifacts: [], - promptQueue: [], - timeline: { events: [], hasMore: false, cursor: null }, - }) - ); + ...(["off", "shadow", "on"] as const).map((mode) => ({ + permissions: undefined, + exposed: false, + mode, + visibility: "private" as const, + })), + ])( + "scopes $mode $visibility snapshot sandbox locations to sandbox access ($exposed)", + async (input) => { + const fetch = vi.fn(async () => + Response.json({ + session: { + id: "session-1", + title: "Session", + repoOwner: "acme", + repoName: "web", + baseBranch: "main", + branchName: "feature", + status: "active", + sandboxStatus: "ready", + messageCount: 0, + createdAt: 1, + codeServerUrl: "https://code.example", + vncUrl: "https://vnc.example", + ttydUrl: "https://terminal.example", + tunnelUrls: { "3000": "https://app.example" }, + sandboxDashboardUrl: "https://provider.example", + }, + artifacts: [], + promptQueue: [], + timeline: { events: [], hasMore: false, cursor: null }, + }) + ); - const response = await dispatch( - new Request("https://test.local/sessions/session-1"), - createEnv(fetch, { permissions: input.permissions }) - ); - const snapshot = (await response.json()) as { session: Record }; + const response = await dispatch(new Request("https://test.local/sessions/session-1"), { + ...createEnv(fetch, { + permissions: input.permissions, + visibility: input.visibility === "private" ? "private" : undefined, + }), + TEAMS_ENFORCEMENT: input.mode, + }); + const snapshot = (await response.json()) as { session: Record }; - expect(response.status).toBe(200); - if (input.exposed) { - expect(snapshot.session).toHaveProperty("codeServerUrl", "https://code.example"); - } else { - expect(snapshot.session).not.toHaveProperty("codeServerUrl"); - expect(snapshot.session).not.toHaveProperty("vncUrl"); - expect(snapshot.session).not.toHaveProperty("ttydUrl"); - expect(snapshot.session).not.toHaveProperty("tunnelUrls"); - expect(snapshot.session).not.toHaveProperty("sandboxDashboardUrl"); + expect(response.status).toBe(200); + if (input.exposed) { + expect(snapshot.session).toHaveProperty("codeServerUrl", "https://code.example"); + } else { + expect(snapshot.session).not.toHaveProperty("codeServerUrl"); + expect(snapshot.session).not.toHaveProperty("vncUrl"); + expect(snapshot.session).not.toHaveProperty("ttydUrl"); + expect(snapshot.session).not.toHaveProperty("tunnelUrls"); + expect(snapshot.session).not.toHaveProperty("sandboxDashboardUrl"); + } } - }); + ); it("forwards event query strings through the session runtime dependency", async () => { const requests: Request[] = []; @@ -729,7 +756,12 @@ describe("session runtime proxy routes", () => { }); it("rejects a malformed budget body before reading the session", async () => { - const get = vi.spyOn(SessionIndexStore.prototype, "get"); + const get = vi.spyOn(SessionIndexStore.prototype, "get").mockResolvedValue({ + id: "session-1", + userId: "user-1", + ownerTeamId: null, + visibility: "workspace", + } as Awaited>); const fetch = vi.fn(async () => Response.json({ maxSessionCostUsd: 20 })); const response = await dispatch( @@ -743,7 +775,7 @@ describe("session runtime proxy routes", () => { expect(response.status).toBe(400); await expect(response.json()).resolves.toEqual({ error: "Invalid budget request" }); - expect(get).not.toHaveBeenCalled(); + expect(get).toHaveBeenCalledOnce(); expect(fetch).not.toHaveBeenCalled(); }); }); diff --git a/packages/control-plane/src/routes/session-runtime-proxy.ts b/packages/control-plane/src/routes/session-runtime-proxy.ts index d0e94129f1..b77f2e4ff1 100644 --- a/packages/control-plane/src/routes/session-runtime-proxy.ts +++ b/packages/control-plane/src/routes/session-runtime-proxy.ts @@ -15,6 +15,7 @@ import { import { z } from "zod"; import { UserStore } from "../db/user-store"; import { SessionIndexStore } from "../db/session-index"; +import { checkSessionAccess } from "@open-inspect/shared"; import type { SubscriptionProviderId } from "@open-inspect/shared/types/provider-accounts"; import { SessionInternalPaths, type SessionInternalPath } from "../session/contracts"; import type { Env } from "../types"; @@ -23,7 +24,7 @@ import { GITHUB_SANDBOX_FALLBACK_ROUTE, GITHUB_USER_OR_SERVICE_ROUTE, NO_AUTHORIZATION, - requirePermission, + requireSession, SCM_AGNOSTIC_SANDBOX_FALLBACK_ROUTE, SCM_AGNOSTIC_HANDLER_AUTHENTICATED_ROUTE, SCM_AGNOSTIC_SANDBOX_ROUTE, @@ -174,9 +175,12 @@ async function handleSessionSnapshot( const parsed = sessionSnapshotSchema.safeParse(await response.json().catch(() => null)); if (!parsed.success) return error("Invalid session snapshot", 502); - const snapshot = ctx.authorization?.permissions.includes("sessions.sandbox_access") - ? parsed.data - : redactSessionSnapshotSandboxAccess(parsed.data); + const admission = ctx.sessionAdmission; + const sandboxAllowed = + admission && (admission.row.visibility === "private" || ctx.teamsEnforcementMode === "on") + ? checkSessionAccess(admission.viewer, admission.row, "sandbox").allowed + : ctx.authorization?.permissions.includes("sessions.sandbox_access"); + const snapshot = sandboxAllowed ? parsed.data : redactSessionSnapshotSandboxAccess(parsed.data); const headers = new Headers(response.headers); headers.delete("Content-Length"); return Response.json(snapshot, { headers }); @@ -305,7 +309,7 @@ async function handleBudgetUpdate( /** Every proxied session operation, by the name its route is known by. */ const LIFECYCLE = admit({ ...GITHUB_USER_OR_SERVICE_ROUTE, - authorization: requirePermission("sessions.lifecycle"), + authorization: requireSession("lifecycle"), }); export const sessionRuntimeProxyRoutes = new Hono(); @@ -314,20 +318,20 @@ sessionRuntimeProxyRoutes.get( "/sessions/:id/sandbox-access", admit({ ...SCM_AGNOSTIC_HUMAN_USER_ROUTE, - authorization: requirePermission("sessions.sandbox_access"), + authorization: requireSession("sandbox"), }), (c) => dispatchSession(c, simpleProxy({ internalPath: SessionInternalPaths.sandboxAccess })) ); sessionRuntimeProxyRoutes.get( "/sessions/:id", - admit({ ...SCM_AGNOSTIC_HUMAN_USER_ROUTE, authorization: requirePermission("sessions.read") }), + admit({ ...SCM_AGNOSTIC_HUMAN_USER_ROUTE, authorization: requireSession("read") }), (c) => dispatchSession(c, handleSessionSnapshot) ); sessionRuntimeProxyRoutes.post( "/sessions/:id/stop", admit({ ...GITHUB_USER_OR_SERVICE_ROUTE, - authorization: requirePermission("sessions.lifecycle", { + authorization: requireSession("lifecycle", { actorlessGrants: [{ service: "linear-bot" }], }), }), @@ -346,7 +350,7 @@ sessionRuntimeProxyRoutes.get( "/sessions/:id/events", admit({ ...GITHUB_USER_OR_SERVICE_ROUTE, - authorization: requirePermission("sessions.read", { + authorization: requireSession("read", { actorlessGrants: [{ service: "slack-bot" }, { service: "linear-bot" }], }), }), @@ -360,7 +364,7 @@ sessionRuntimeProxyRoutes.get( "/sessions/:id/artifacts", admit({ ...GITHUB_USER_OR_SERVICE_ROUTE, - authorization: requirePermission("sessions.read", { + authorization: requireSession("read", { actorlessGrants: [{ service: "slack-bot" }, { service: "linear-bot" }], }), }), @@ -368,20 +372,20 @@ sessionRuntimeProxyRoutes.get( ); sessionRuntimeProxyRoutes.get( "/sessions/:id/participants", - admit({ ...GITHUB_USER_OR_SERVICE_ROUTE, authorization: requirePermission("sessions.read") }), + admit({ ...GITHUB_USER_OR_SERVICE_ROUTE, authorization: requireSession("read") }), (c) => dispatchSession(c, simpleProxy({ internalPath: SessionInternalPaths.participants })) ); sessionRuntimeProxyRoutes.get( "/sessions/:id/participant-profiles", admit({ ...SCM_AGNOSTIC_USER_OR_SERVICE_ROUTE, - authorization: requirePermission("sessions.read"), + authorization: requireSession("read"), }), (c) => dispatchSession(c, handleParticipantProfiles) ); sessionRuntimeProxyRoutes.get( "/sessions/:id/messages", - admit({ ...GITHUB_USER_OR_SERVICE_ROUTE, authorization: requirePermission("sessions.read") }), + admit({ ...GITHUB_USER_OR_SERVICE_ROUTE, authorization: requireSession("read") }), (c) => dispatchSession( c, @@ -392,7 +396,7 @@ sessionRuntimeProxyRoutes.post( "/sessions/:id/pr", admit({ ...GITHUB_SANDBOX_FALLBACK_ROUTE, - authorization: requirePermission("sessions.collaborate"), + authorization: requireSession("collaborate"), }), (c) => dispatchSession(c, handleCreatePR) ); @@ -419,7 +423,7 @@ sessionRuntimeProxyRoutes.get( "/sessions/:id/tunnel-urls", admit({ ...SCM_AGNOSTIC_SANDBOX_FALLBACK_ROUTE, - authorization: requirePermission("sessions.sandbox_access"), + authorization: requireSession("sandbox"), }), (c) => dispatchSession( @@ -440,7 +444,7 @@ sessionRuntimeProxyRoutes.patch( "/sessions/:id/budget", admit({ ...SCM_AGNOSTIC_HUMAN_USER_ROUTE, - authorization: requirePermission("sessions.lifecycle"), + authorization: requireSession("lifecycle"), }), (c) => dispatchSession(c, handleBudgetUpdate) ); diff --git a/packages/control-plane/src/routes/session-skills.ts b/packages/control-plane/src/routes/session-skills.ts index b8488a31d6..40167d07ff 100644 --- a/packages/control-plane/src/routes/session-skills.ts +++ b/packages/control-plane/src/routes/session-skills.ts @@ -9,7 +9,7 @@ import { error, json, NO_AUTHORIZATION, - requirePermission, + requireSession, SCM_AGNOSTIC_SANDBOX_ROUTE, SCM_AGNOSTIC_HUMAN_USER_ROUTE, type SandboxRouteContext, @@ -92,7 +92,7 @@ export const sessionSkillRoutes = new Hono(); sessionSkillRoutes.get( "/sessions/:id/skills", - admit({ ...SCM_AGNOSTIC_HUMAN_USER_ROUTE, authorization: requirePermission("sessions.read") }), + admit({ ...SCM_AGNOSTIC_HUMAN_USER_ROUTE, authorization: requireSession("read") }), (c) => dispatch(c, handleSessionSkillsView) ); diff --git a/packages/control-plane/src/routes/session-ws-token.ts b/packages/control-plane/src/routes/session-ws-token.ts index 79361045a1..d1a41e6c55 100644 --- a/packages/control-plane/src/routes/session-ws-token.ts +++ b/packages/control-plane/src/routes/session-ws-token.ts @@ -2,10 +2,9 @@ import { Hono } from "hono"; import { admit } from "../routing/admit"; import type { ControlPlaneHonoEnv } from "../routing/hono-env"; import { applyIdentityEnforcement } from "../routing/identity-enforcement"; -import { SESSION_WEBSOCKET_CONNECT_PERMISSION } from "@open-inspect/shared/rbac"; import { SessionInternalPaths, sessionScmDisplayFieldsSchema } from "../session/contracts"; import type { Env } from "../types"; -import { error, GITHUB_USER_OR_SERVICE_ROUTE, requirePermission } from "./shared"; +import { error, GITHUB_USER_OR_SERVICE_ROUTE, requireSession } from "./shared"; import { parseJsonBody } from "./body"; import { dispatchSession, type SessionRouteContext } from "./session-route"; @@ -55,7 +54,7 @@ sessionWsTokenRoutes.post( "/sessions/:id/ws-token", admit({ ...GITHUB_USER_OR_SERVICE_ROUTE, - authorization: requirePermission(SESSION_WEBSOCKET_CONNECT_PERMISSION), + authorization: requireSession("read"), }), (c) => dispatchSession(c, handleSessionWsToken) ); diff --git a/packages/control-plane/src/routes/shared.ts b/packages/control-plane/src/routes/shared.ts index 51fc063907..226c939929 100644 --- a/packages/control-plane/src/routes/shared.ts +++ b/packages/control-plane/src/routes/shared.ts @@ -10,6 +10,7 @@ import type { Logger } from "../logger"; import type { PermissionId } from "@open-inspect/shared/rbac"; import type { ServiceName } from "@open-inspect/shared/service-auth"; import type { TeamCapabilities } from "@open-inspect/shared/types/team-access"; +import type { SessionAction } from "@open-inspect/shared"; import { createSourceControlProviderFromEnv, SourceControlProviderError, @@ -45,7 +46,8 @@ export type RouteAuthorizationRequirement = operation: "manage" | "trigger"; automationIdParam: string; } - | { kind: "team"; teamIdParam: string; need: keyof TeamCapabilities | "read" }; + | { kind: "team"; teamIdParam: string; need: keyof TeamCapabilities | "read" } + | { kind: "session"; sessionIdParam: string; action: SessionAction }; type BotServiceName = Exclude; const DEFAULT_AUDIT_ALLOWED = false; @@ -141,6 +143,7 @@ const AUDITED_ALLOWED_PERMISSIONS = new Set([ ]); function auditsAllowedRequirement(requirement: RouteAuthorizationRequirement): boolean { + if (requirement.kind === "session") return requirement.action !== "read"; if (requirement.kind === "permission") { return AUDITED_ALLOWED_PERMISSIONS.has(requirement.permission); } @@ -193,6 +196,25 @@ export function requireTeam( }; } +export function sessionRequirement( + action: SessionAction, + sessionIdParam = "id" +): RouteAuthorizationRequirement { + return { kind: "session", sessionIdParam, action }; +} + +export function requireSession( + action: SessionAction, + options?: { sessionIdParam?: string; actorlessGrants?: readonly ActorlessServiceGrant[] } +): RouteAuthorization { + return { + kind: "active-user", + allOf: [sessionRequirement(action, options?.sessionIdParam)], + service: { kind: "actor", actorlessGrants: options?.actorlessGrants }, + auditAllowed: action !== "read", + }; +} + /** Require an active user to satisfy every supplied authorization requirement. */ export function requireAll(...allOf: readonly RouteAuthorizationRequirement[]): RouteAuthorization { return { diff --git a/packages/control-plane/src/routes/slack-notify.ts b/packages/control-plane/src/routes/slack-notify.ts index 91515061d8..368a56e1de 100644 --- a/packages/control-plane/src/routes/slack-notify.ts +++ b/packages/control-plane/src/routes/slack-notify.ts @@ -20,12 +20,7 @@ import { IntegrationSettingsStore, resolveSlackSettings } from "../db/integratio import { SessionIndexStore } from "../db/session-index"; import { createLogger } from "../logger"; import type { Env } from "../types"; -import { - GITHUB_SANDBOX_FALLBACK_ROUTE, - json, - requirePermission, - type RequestContext, -} from "./shared"; +import { GITHUB_SANDBOX_FALLBACK_ROUTE, json, requireSession, type RequestContext } from "./shared"; const logger = createLogger("slack-notify"); @@ -317,7 +312,7 @@ slackNotifyRoutes.post( "/sessions/:id/slack-notify", admit({ ...GITHUB_SANDBOX_FALLBACK_ROUTE, - authorization: requirePermission("sessions.collaborate"), + authorization: requireSession("collaborate"), }), (c) => dispatch(c, handleSlackNotify) ); diff --git a/packages/control-plane/src/routing/hono-app.ts b/packages/control-plane/src/routing/hono-app.ts index 9cdb99a170..8dc9b2650c 100644 --- a/packages/control-plane/src/routing/hono-app.ts +++ b/packages/control-plane/src/routing/hono-app.ts @@ -179,13 +179,18 @@ export function createControlPlaneApp( } if (!unexpected) logRequest(c.res, context, method, pathname, startedAt); - if (shouldAuditAllowedDecision(result.decision)) { + const decision = { + ...result.decision, + ...(context.shadowSessionDenial ? { shadowReason: context.shadowSessionDenial } : {}), + ...(context.shadowBatchDenials?.length ? { shadowDenials: context.shadowBatchDenials } : {}), + }; + if (shouldAuditAllowedDecision(decision)) { await auditRouteAuthorizationDecision({ ctx: context, method, path: pathname, response: c.res, - decision: result.decision, + decision, }); } replaceResponse(c, finalizeRouteResponse(c.res, policy, context)); diff --git a/packages/control-plane/src/routing/route-admission.ts b/packages/control-plane/src/routing/route-admission.ts index ff0ebc008e..00ac3f6a86 100644 --- a/packages/control-plane/src/routing/route-admission.ts +++ b/packages/control-plane/src/routing/route-admission.ts @@ -19,6 +19,8 @@ import type { } from "../authorization/request-audit"; import { AuthorizationError, AuthorizationService } from "../authorization/service"; import { serviceAllowsPermission } from "../authorization/service-permissions"; +import { evaluateSessionAdmission, viewerFromContext } from "../authorization/session-admission"; +import { legacyPermissionForAction } from "../authorization/teams-enforcement"; import { AutomationStore } from "../db/automation-store"; import { PersonalAccessTokenStore } from "../db/personal-access-tokens"; import { TeamStore } from "../db/teams"; @@ -368,7 +370,12 @@ function enforceStaticServicePermissionCeiling( if (policy.authorization.kind !== "active-user") return null; for (const requirement of policy.authorization.allOf) { - const permission = requirement.kind === "permission" ? requirement.permission : null; + const permission = + requirement.kind === "permission" + ? requirement.permission + : requirement.kind === "session" + ? legacyPermissionForAction(requirement.action) + : null; if (permission && !serviceAllowsPermission(principal.service, permission)) { return authorizationDenial( json({ error: "Forbidden", code: "service_capability_required" }, 403), @@ -632,23 +639,24 @@ async function enforceTeamRequirement( const teamId = params[requirement.teamIdParam]; if (!teamId) return { response: json({ error: "Invalid team route" }, 400) }; try { - const authorization = ctx.authorization; - if (!authorization) throw new Error("Missing request authorization"); const team = await new TeamStore(ctx.db).getById(teamId); if (!team) return { response: error("Team not found", 404) }; const memberships = new TeamMembershipStore(ctx.db); - const viewerMemberships = await memberships.listForUser(ctx.principal.userId); + const viewer = viewerFromContext( + ctx, + (ctx.sessionMemberships ??= await memberships.listForUser(ctx.principal.userId)) + ); + if (viewer.kind !== "user") throw new Error("Missing team viewer"); const access = resolveTeamAccess( { - userId: ctx.principal.userId, - roleKey: authorization.role.key, - memberships: viewerMemberships, + userId: viewer.userId, + roleKey: viewer.roleKey, + memberships: viewer.memberships, }, { ...team, leadCount: await memberships.countLeads(teamId) } ); - const isAdmin = - authorization.role.key === "owner" || authorization.role.key === "administrator"; - const visible = isAdmin || viewerMemberships.has(teamId); + const isAdmin = viewer.roleKey === "owner" || viewer.roleKey === "administrator"; + const visible = isAdmin || viewer.memberships.has(teamId); if (!visible && requirement.need !== "canJoin") return { response: error("Team not found", 404) }; if (requirement.need !== "read" && !access[requirement.need]) { @@ -676,6 +684,59 @@ async function enforceTeamRequirement( } } +async function enforceSessionRequirement( + requirement: Extract, + params: RouteParams, + env: Env, + ctx: RequestContext, + evidence: AuthorizationEvidence +): Promise { + const sessionId = params[requirement.sessionIdParam]; + if (!sessionId) return { response: json({ error: "Invalid session route" }, 400) }; + try { + const result = await evaluateSessionAdmission( + ctx, + env, + sessionId, + requirement.action, + requirement.sessionIdParam === "childId" ? "child" : "session" + ); + if (result.kind === "not_found") { + return authorizationDenial( + error("Session not found", 404), + evidence, + requirement, + "session_not_visible", + "Session not found" + ); + } + if (result.kind === "action_denied") { + return authorizationDenial( + json( + { error: "Forbidden", code: "session_action_denied", reason_code: result.reason }, + 403 + ), + evidence, + requirement, + result.reason, + "Forbidden" + ); + } + if (result.legacyPermission) { + const legacy = await enforcePermissionRequirement( + { kind: "permission", permission: result.legacyPermission }, + ctx, + evidence + ); + if (legacy) return legacy; + } + evidence.requirements.push(requirement); + return null; + } catch { + return authorizationUnavailable(); + } +} + function allowed( policy: RouteAdmissionPolicy, admission: AllowedAuthorizationDecision["admission"], @@ -757,6 +818,9 @@ async function enforceRouteAuthorization( case "team": failure = await enforceTeamRequirement(requirement, params, ctx, evidence); break; + case "session": + failure = await enforceSessionRequirement(requirement, params, env, ctx, evidence); + break; } if (failure) return resultForFailure(failure); } diff --git a/packages/control-plane/src/sandbox/client.test.ts b/packages/control-plane/src/sandbox/client.test.ts index 2f122a60f1..8ddcffc694 100644 --- a/packages/control-plane/src/sandbox/client.test.ts +++ b/packages/control-plane/src/sandbox/client.test.ts @@ -137,6 +137,87 @@ describe("ModalClient", () => { }); }); + it("resolves a VM using only generation identity and preserves typed errors", async () => { + const fetchMock = vi + .spyOn(globalThis, "fetch") + .mockResolvedValueOnce( + Response.json({ + success: true, + data: { + sandbox_id: "generation", + modal_object_id: "sb-real", + sandbox_backend: "modal-vm", + code_server_url: "https://editor.example", + code_server_password: "password", + }, + }) + ) + .mockResolvedValueOnce(Response.json({ detail: "not_visible" }, { status: 409 })); + const client = createModalClient("secret", "acme"); + expect( + await client.resolveVmSandbox({ sessionId: "session", sandboxId: "generation" }) + ).toMatchObject({ + sandboxId: "generation", + modalObjectId: "sb-real", + codeServerPassword: "password", + }); + expect(String(fetchMock.mock.calls[0][0])).toBe( + "https://acme--open-inspect-api-resolve-vm-sandbox.modal.run" + ); + expect(JSON.parse(fetchMock.mock.calls[0][1]?.body as string)).toEqual({ + session_id: "session", + sandbox_id: "generation", + }); + await expect( + client.resolveVmSandbox({ sessionId: "session", sandboxId: "generation" }) + ).rejects.toMatchObject({ + status: 409, + detail: "not_visible", + }); + }); + + it.each([ + ["server error", Response.json({ detail: "Internal server error" }, { status: 500 })], + ["invalid success", Response.json({ success: true, data: {} })], + ["truncated success", new Response("{", { status: 200 })], + ])("types a VM startup %s as an unknown outcome after dispatch", async (_case, response) => { + vi.spyOn(globalThis, "fetch").mockResolvedValue(response); + const client = createModalClient("secret", "acme"); + await expect( + client.createSandbox({ + sessionId: "session", + sandboxId: "generation", + sandboxBackend: "modal-vm", + repoOwner: null, + repoName: null, + controlPlaneUrl: "https://control.test", + sandboxAuthToken: "token", + harness: "opencode", + }) + ).rejects.toMatchObject({ name: "ModalVmStartupError", outcome: "unknown" }); + }); + + it("types a VM launch-window rejection without treating it as unknown", async () => { + vi.spyOn(globalThis, "fetch").mockResolvedValue( + Response.json({ detail: "window_closed" }, { status: 409 }) + ); + await expect( + createModalClient("secret", "acme").restoreSandbox({ + snapshotImageId: "image", + sessionId: "session", + sandboxId: "generation", + sandboxBackend: "modal-vm", + sandboxAuthToken: "token", + controlPlaneUrl: "https://control.test", + repoOwner: null, + repoName: null, + harness: "opencode", + provider: "anthropic", + model: "test", + }) + ).rejects.toMatchObject({ name: "ModalVmStartupError", outcome: "window_closed" }); + }); + it("times out image-build creation when response headers stall", async () => { vi.useFakeTimers(); let markFetchStarted!: () => void; diff --git a/packages/control-plane/src/sandbox/client.ts b/packages/control-plane/src/sandbox/client.ts index ab155d4327..e552187299 100644 --- a/packages/control-plane/src/sandbox/client.ts +++ b/packages/control-plane/src/sandbox/client.ts @@ -65,6 +65,12 @@ const restoreSandboxModalResponseSchema = z.object({ }), }); +const resolveVmSandboxModalResponseSchema = restoreSandboxModalResponseSchema.extend({ + data: restoreSandboxModalResponseSchema.shape.data.extend({ + modal_object_id: z.string().min(1), + }), +}); + const snapshotSandboxModalResponseSchema = z.object({ success: z.literal(true), data: z.object({ @@ -234,6 +240,13 @@ export interface RestoreSandboxResponse { tunnelUrls?: Record; } +export interface ResolveVmSandboxRequest { + sessionId: string; + sandboxId: string; +} + +export type ResolveVmSandboxResponse = RestoreSandboxResponse & { modalObjectId: string }; + export interface SnapshotSandboxRequest { providerObjectId: string; sessionId: string; @@ -317,6 +330,23 @@ export class ModalApiError extends Error { } } +export type ModalVmStartupOutcome = + | "unknown" + | "not_visible" + | "other_generation" + | "window_closed" + | "race_pending"; + +export class ModalVmStartupError extends Error { + constructor( + public readonly outcome: ModalVmStartupOutcome, + public readonly cause: Error + ) { + super(cause.message); + this.name = "ModalVmStartupError"; + } +} + /** * Modal sandbox API client. * @@ -328,6 +358,7 @@ export class ModalClient { private snapshotVmSandboxUrl: string; private snapshotBuildSandboxUrl: string; private restoreSandboxUrl: string; + private resolveVmSandboxUrl: string; private stopSandboxUrl: string; private createImageBuildSandboxUrl: string; private startImageBuildSandboxUrl: string; @@ -342,40 +373,67 @@ export class ModalClient { schema: z.ZodType, correlation: CorrelationContext | undefined, callerSignal: AbortSignal | undefined, - onResponse: (status: number) => void + onResponse: (status: number) => void, + vmStartup = false ): Promise { const headers = await this.getPostHeaders(correlation); - return withRequestDeadline("Modal", endpoint, deadlineMs, callerSignal, async (signal) => { - const response = await fetch(url, { - method: "POST", - headers, - signal, - body: JSON.stringify(body), - }); - onResponse(response.status); - if (!response.ok) { - const text = await response.text(); - let body: unknown; - try { - body = JSON.parse(text); - } catch { - // Non-JSON provider responses still retain their status and raw text. + const payload = JSON.stringify(body); + try { + return await withRequestDeadline( + "Modal", + endpoint, + deadlineMs, + callerSignal, + async (signal) => { + const response = await fetch(url, { + method: "POST", + headers, + signal, + body: payload, + }); + onResponse(response.status); + if (!response.ok) { + const text = await response.text(); + let body: unknown; + try { + body = JSON.parse(text); + } catch { + // Non-JSON provider responses still retain their status and raw text. + } + const detail = + body !== null && + typeof body === "object" && + "detail" in body && + typeof body.detail === "string" + ? body.detail + : undefined; + throw new ModalApiError( + `Modal API error: ${response.status} ${text}`, + response.status, + detail + ); + } + return parseModalApiResponse(schema, await response.json()); } - const detail = - body !== null && - typeof body === "object" && - "detail" in body && - typeof body.detail === "string" - ? body.detail - : undefined; - throw new ModalApiError( - `Modal API error: ${response.status} ${text}`, - response.status, - detail - ); + ); + } catch (error) { + if (!vmStartup) throw error; + if (error instanceof ModalApiError) { + const detail = error.detail; + if ( + detail === "not_visible" || + detail === "other_generation" || + detail === "window_closed" || + detail === "race_pending" + ) + throw new ModalVmStartupError(detail, error); + if (error.status < 500) throw error; } - return parseModalApiResponse(schema, await response.json()); - }); + throw new ModalVmStartupError( + "unknown", + error instanceof Error ? error : new Error(String(error)) + ); + } } constructor(secret: string, workspace: string, environmentWebSuffix?: string, apiUrl?: string) { @@ -393,6 +451,7 @@ export class ModalClient { this.snapshotVmSandboxUrl = url("api-snapshot-vm-sandbox"); this.snapshotBuildSandboxUrl = url("api-snapshot-build-sandbox"); this.restoreSandboxUrl = url("api-restore-sandbox"); + this.resolveVmSandboxUrl = url("api-resolve-vm-sandbox"); this.stopSandboxUrl = url("api-stop-sandbox"); this.createImageBuildSandboxUrl = url("api-create-build-sandbox"); this.startImageBuildSandboxUrl = url("api-start-build-sandbox"); @@ -467,7 +526,8 @@ export class ModalClient { createSandboxModalResponseSchema, correlation, request.signal, - (status) => (httpStatus = status) + (status) => (httpStatus = status), + request.sandboxBackend === "modal-vm" ); outcome = "success"; @@ -534,7 +594,8 @@ export class ModalClient { restoreSandboxModalResponseSchema, correlation, request.signal, - (status) => (httpStatus = status) + (status) => (httpStatus = status), + request.sandboxBackend === "modal-vm" ); outcome = "success"; @@ -564,6 +625,34 @@ export class ModalClient { } } + /** Lookup-only recovery of a generation's named Modal VM allocation. */ + async resolveVmSandbox( + request: ResolveVmSandboxRequest, + correlation?: CorrelationContext + ): Promise { + const result = await this.postJson( + this.resolveVmSandboxUrl, + "resolveVmSandbox", + MODAL_SANDBOX_START_REQUEST_DEADLINE_MS, + { session_id: request.sessionId, sandbox_id: request.sandboxId }, + resolveVmSandboxModalResponseSchema, + correlation, + undefined, + () => {} + ); + return { + sandboxId: result.data.sandbox_id, + modalObjectId: result.data.modal_object_id, + sandboxBackend: result.data.sandbox_backend, + codeServerUrl: result.data.code_server_url ?? undefined, + codeServerPassword: result.data.code_server_password ?? undefined, + vncUrl: result.data.vnc_url ?? undefined, + vncPassword: result.data.vnc_password ?? undefined, + ttydUrl: result.data.ttyd_url ?? undefined, + tunnelUrls: result.data.tunnel_urls ?? undefined, + }; + } + /** Trigger a filesystem snapshot for a sandbox object. */ async snapshotSandbox( request: SnapshotSandboxRequest, diff --git a/packages/control-plane/src/sandbox/lifecycle/manager.test.ts b/packages/control-plane/src/sandbox/lifecycle/manager.test.ts index 565e2482dc..e09203ce83 100644 --- a/packages/control-plane/src/sandbox/lifecycle/manager.test.ts +++ b/packages/control-plane/src/sandbox/lifecycle/manager.test.ts @@ -202,9 +202,10 @@ async function expectEarlyBridgeStartup(kind: ProviderStartupKind): Promise { function fixture( provider = createMockProvider(), - sandbox = createMockSandbox({ status: "stopped" }) + sandbox = createMockSandbox({ status: "stopped" }), + session = createMockSession() ) { - const storage = createMockStorage(createMockSession(), sandbox); + const storage = createMockStorage(session, sandbox); const sockets = createMockWebSocketManager(); const shutdown = { ...createUnmanagedShutdown(), @@ -336,6 +337,130 @@ describe("final graceful shutdown lifecycle integration", () => { expect(f.provider.createSandbox).not.toHaveBeenCalled(); }); + it.each(["connect timeout", "fatal runtime error", "boot budget"] as const)( + "holds a resumed retained source after a %s instead of deleting it", + async (failure) => { + vi.useFakeTimers(); + try { + const resumeSandbox = vi.fn(async () => ({ + success: true as const, + providerObjectId: "retained-source", + lifetime: noLifetime(), + ttydUrl: "https://terminal.test/resumed", + })); + const stopSandbox = vi.fn(async () => ({ success: true })); + const f = fixture( + createMockProvider({ + resumeSandbox, + stopSandbox, + capabilities: { supportsPersistentResume: true, supportsExplicitStop: true }, + }), + createMockSandbox({ + status: "stopped", + modal_object_id: "retained-source", + // The repository clears this on resume; the mock does not. + last_heartbeat: null, + ttyd_token: await mintJwt( + { exp: Math.floor(Date.now() / 1000) - 1 }, + "sandbox-auth-token" + ), + }), + createMockSession({ sandbox_settings: JSON.stringify({ terminalEnabled: true }) }) + ); + const saved = withSavedState(f, "retained"); + + await f.manager.spawnSandbox(); + expect(saved.read().phase).toBe("running"); + const row = f.storage.getSandbox()!; + if (failure === "fatal runtime error") { + row.last_heartbeat = Date.now(); + expect(await f.manager.terminateFailedSandbox("runtime failed")).toBe(false); + } else { + vi.advanceTimersByTime( + failure === "connect timeout" + ? DEFAULT_LIFECYCLE_CONFIG.connectingTimeout.timeoutMs + 1 + : DEFAULT_LIFECYCLE_CONFIG.bootBudget.timeoutMs + 1 + ); + if (failure === "boot budget") row.last_heartbeat = Date.now(); + expect(await f.manager.handleShutdownAlarm()).toBe("continue"); + await f.manager.handleAlarm(); + } + + // Neither deleted nor fenced: the source is the only copy of the workspace. + expect(stopSandbox).not.toHaveBeenCalled(); + expect(row).toMatchObject({ + status: "failed", + modal_object_id: "retained-source", + fenced: 0, + }); + expect(saved.read()).toMatchObject({ + phase: "unknown", + receipt: { kind: "retained", artifactId: "retained-source" }, + }); + await f.manager.spawnSandbox(); + expect(f.provider.createSandbox).not.toHaveBeenCalled(); + + await saved.shutdown.recover("restore_saved"); + expect(stopSandbox).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining({ providerObjectId: "retained-source", intent: "preserve" }) + ); + await f.manager.spawnSandbox(); + expect(resumeSandbox).toHaveBeenCalledTimes(2); + expect(saved.read().phase).toBe("running"); + expect(f.provider.createSandbox).not.toHaveBeenCalled(); + } finally { + vi.useRealTimers(); + } + } + ); + + it("keeps restore available after a later ordinary resume of the retained source fails mid-resume", async () => { + let finishResume!: (result: ResumeResult) => void; + const resumed = { + success: true as const, + providerObjectId: "retained-source", + lifetime: noLifetime(), + }; + const resumeSandbox = vi + .fn>() + .mockResolvedValueOnce(resumed) + .mockReturnValueOnce(new Promise((resolve) => (finishResume = resolve))) + .mockResolvedValue(resumed); + const stopSandbox = vi.fn(async () => ({ success: true })); + const f = fixture( + createMockProvider({ + resumeSandbox, + stopSandbox, + capabilities: { supportsPersistentResume: true, supportsExplicitStop: true }, + }), + createMockSandbox({ status: "stopped", modal_object_id: "retained-source" }) + ); + const saved = withSavedState(f, "retained"); + await f.manager.spawnSandbox(); + // A heartbeat timeout preserve-stops it; the shutdown record stays running. + const row = f.storage.getSandbox()!; + row.status = "stopped"; + + const ordinaryResume = f.manager.spawnSandbox(); + await vi.waitFor(() => expect(resumeSandbox).toHaveBeenCalledTimes(2)); + row.last_heartbeat = Date.now(); + expect(await f.manager.terminateFailedSandbox("runtime failed")).toBe(false); + finishResume(resumed); + await ordinaryResume; + + expect(stopSandbox).not.toHaveBeenCalled(); + expect(saved.read()).toMatchObject({ phase: "unknown", providerObjectId: "retained-source" }); + expect(saved.shutdown.snapshot()?.availableRecoveryActions).toContain("restore_saved"); + await saved.shutdown.recover("restore_saved"); + expect(stopSandbox).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining({ providerObjectId: "retained-source", intent: "preserve" }) + ); + await f.manager.spawnSandbox(); + expect(resumeSandbox).toHaveBeenCalledTimes(3); + expect(saved.read().phase).toBe("running"); + expect(f.provider.createSandbox).not.toHaveBeenCalled(); + }); + it("allows only explicit retry of an ambiguous snapshot restore from a retired source", async () => { const restoreFromSnapshot = vi .fn>() @@ -1981,7 +2106,7 @@ describe("SandboxLifecycleManager", () => { }); it.each(["missing", "expired"] as const)( - "replaces a resumable sandbox when its terminal token is %s", + "keeps a resumed sandbox without terminal access when its terminal token is %s", async (credentialState) => { const ttydToken = credentialState === "expired" @@ -2003,7 +2128,6 @@ describe("SandboxLifecycleManager", () => { providerObjectId: "replacement-provider-obj", createdAt: Date.now(), lifetime: noLifetime(), - ttydUrl: "https://terminal.test/replacement", })); const resumeSandbox = vi.fn(async () => ({ success: true as const, @@ -2032,22 +2156,19 @@ describe("SandboxLifecycleManager", () => { await manager.spawnSandbox(); - expect(resumeSandbox).toHaveBeenCalled(); - expect(stopSandbox).toHaveBeenCalledWith( - expect.objectContaining({ - providerObjectId: "old-provider-obj", - reason: "respawn", - }) - ); - expect(createSandbox).toHaveBeenCalledWith( - expect.objectContaining({ sandboxSettings: { terminalEnabled: true } }) - ); - expect(sandbox.ttyd_token).not.toBeNull(); - expect(sandbox.ttyd_token).not.toBe(ttydToken); + // The stopped sandbox is the only copy of the workspace; losing the + // terminal must not delete it. + expect(resumeSandbox).toHaveBeenCalledOnce(); + expect(stopSandbox).not.toHaveBeenCalled(); + expect(createSandbox).not.toHaveBeenCalled(); + expect(sandbox.modal_object_id).toBe("old-provider-obj"); + expect(sandbox.status).toBe("connecting"); + expect(sandbox.ttyd_url).toBeNull(); + expect(sandbox.ttyd_token).toBeNull(); } ); - it("replaces a resumed sandbox after its initial terminal preview could not be issued", async () => { + it("keeps resuming a sandbox whose initial terminal preview could not be issued", async () => { const sandbox = createMockSandbox({ status: "pending", created_at: Date.now() - 60_000, @@ -2059,28 +2180,24 @@ describe("SandboxLifecycleManager", () => { createMockSession({ sandbox_settings: JSON.stringify({ terminalEnabled: true }) }), sandbox ); - let createCount = 0; - const createSandbox = vi.fn(async (config: CreateSandboxConfig) => { - createCount++; - return { - sandboxId: config.sandboxId, - providerObjectId: createCount === 1 ? "initial-provider-obj" : "replacement-provider-obj", - createdAt: Date.now(), - lifetime: noLifetime(), - ...(createCount === 2 ? { ttydUrl: "https://terminal.test/replacement" } : {}), - }; - }); + const createSandbox = vi.fn(async (config: CreateSandboxConfig) => ({ + sandboxId: config.sandboxId, + providerObjectId: "initial-provider-obj", + createdAt: Date.now(), + lifetime: noLifetime(), + })); const resumeSandbox = vi.fn(async () => ({ success: true as const, providerObjectId: "initial-provider-obj", lifetime: noLifetime(), ttydUrl: "https://terminal.test/resumed", })); + const stopSandbox = vi.fn(async () => ({ success: true })); const provider = createMockProvider({ capabilities: { supportsExplicitStop: true, supportsPersistentResume: true }, createSandbox, resumeSandbox, - stopSandbox: vi.fn(async () => ({ success: true })), + stopSandbox, }); const manager = new SandboxLifecycleManager( provider, @@ -2100,9 +2217,59 @@ describe("SandboxLifecycleManager", () => { await manager.spawnSandbox(); expect(resumeSandbox).toHaveBeenCalledOnce(); - expect(createSandbox).toHaveBeenCalledTimes(2); - expect(sandbox.ttyd_url).toBe("https://terminal.test/replacement"); - expect(sandbox.ttyd_token).not.toBeNull(); + expect(createSandbox).toHaveBeenCalledOnce(); + expect(stopSandbox).not.toHaveBeenCalled(); + expect(sandbox.modal_object_id).toBe("initial-provider-obj"); + expect(sandbox.ttyd_url).toBeNull(); + }); + + it("resumes retained saved state without terminal access when its terminal token expired", async () => { + const sandbox = createMockSandbox({ + status: "stopped", + modal_object_id: "retained-source", + ttyd_url: null, + ttyd_token: await mintJwt({ exp: Math.floor(Date.now() / 1000) - 1 }, "sandbox-auth-token"), + }); + const storage = createMockStorage( + createMockSession({ sandbox_settings: JSON.stringify({ terminalEnabled: true }) }), + sandbox + ); + const provider = createMockProvider({ + capabilities: { supportsExplicitStop: true, supportsPersistentResume: true }, + resumeSandbox: vi.fn(async () => ({ + success: true as const, + providerObjectId: "retained-source", + lifetime: noLifetime(), + ttydUrl: "https://terminal.test/resumed", + })), + }); + const shutdown = createUnmanagedShutdown(); + shutdown.startupDecision.mockReturnValue({ + kind: "resume_retained", + providerObjectId: "retained-source", + runtimeVersion: COMPATIBLE_RUNTIME_VERSION, + }); + const manager = new SandboxLifecycleManager( + provider, + storage, + storage, + createMockBroadcaster(), + createMockWebSocketManager(false), + createMockAlarmScheduler(), + createMockIdGenerator(), + shutdown, + createTestConfig() + ); + + await manager.spawnSandbox(); + + // Retrying recovery cannot renew the credential, so holding here would + // leave the saved workspace unrecoverable. + expect(shutdown.holdFailedRecovery).not.toHaveBeenCalled(); + expect(shutdown.recordProviderStartup).toHaveBeenCalledOnce(); + expect(provider.createSandbox).not.toHaveBeenCalled(); + expect(sandbox.modal_object_id).toBe("retained-source"); + expect(sandbox.ttyd_url).toBeNull(); }); it("does not carry a predecessor's runtime version onto a replacement's snapshot", async () => { diff --git a/packages/control-plane/src/sandbox/lifecycle/manager.ts b/packages/control-plane/src/sandbox/lifecycle/manager.ts index 81a85f8b76..285f7c30b7 100644 --- a/packages/control-plane/src/sandbox/lifecycle/manager.ts +++ b/packages/control-plane/src/sandbox/lifecycle/manager.ts @@ -55,6 +55,7 @@ import { DEFAULT_HEARTBEAT_CONFIG, DEFAULT_CONNECTING_TIMEOUT_CONFIG, DEFAULT_BOOT_BUDGET_CONFIG, + PENDING_VM_REFERENCE_MATERIALIZE_BOUND_MS, type CircuitBreakerConfig, type SpawnConfig, } from "./decisions"; @@ -71,7 +72,7 @@ import { type ImageBuildLookup, type SelectedImageBuild, } from "./image-selection"; -import type { AlarmScheduler, SessionWebSocket } from "../../platform-ports"; +import type { AlarmScheduler, BackgroundTasks, SessionWebSocket } from "../../platform-ports"; import { DEFAULT_SANDBOX_STATUS } from "../sandbox-status"; import type { SandboxGeneration, @@ -86,6 +87,9 @@ import type { SandboxPushAdmission, } from "./ports"; import { shutdownPolicyForLaunch, type ShutdownLifecyclePolicy } from "./shutdown-policy"; +import { parsePendingVmReference } from "../providers/pending-vm-reference"; +import { ModalApiError, ModalVmStartupError } from "../client"; +import type { ResolveSandboxResult } from "../provider"; export type { SandboxGeneration, SandboxAlarmResult } from "./ports"; export type { ImageBuildLookup } from "./image-selection"; @@ -97,6 +101,14 @@ const log = createLogger("lifecycle-manager"); const TERMINAL_TOKEN_TTL_SECONDS = 86400; const PROVIDER_REPLACEMENT_STOP_TIMEOUT_MS = 10_000; const REJECTED_ALLOCATION_CLEANUP_RETRY_MS = 30_000; +const VM_RESOLVE_RETRY_MS = 10_000; + +function vmAllocationDetail(error: unknown): string | undefined { + const cause = error instanceof SandboxProviderError ? error.cause : error; + if (cause instanceof ModalVmStartupError) return cause.outcome; + if (cause instanceof ModalApiError) return cause.detail; + return undefined; +} // ==================== Dependency Interfaces ==================== @@ -118,6 +130,12 @@ export interface SandboxShutdownLifecycle { ): Promise<"registered" | "expired" | "superseded">; /** Records the provider-confirmed handle and scheduling lifetime after startup. */ recordProviderStartup(generation: SandboxGeneration, lifetime: SandboxLifetime): Promise; + /** Swap a pending handle after bridge recovery without changing shutdown policy or lifetime. */ + recordResolvedProviderHandle?( + generation: SandboxGeneration, + expectedReference: string, + providerObjectId: string + ): void; /** Blocks generic destructive lifecycle work while shutdown or capture ownership is unresolved. */ isHolding(): boolean; /** Tells a runtime refused at reconnect to retry while a capture needs its sandbox. */ @@ -134,6 +152,8 @@ export interface SandboxShutdownLifecycle { ): Promise; /** Decides startup without exposing the coordinator's persisted receipt representation. */ startupDecision(): SandboxStartupDecision; + /** Holds a failed boot of the retained source, which deleting would lose; false for other objects. */ + holdFailedRetainedBoot(error: string, generation: SandboxGeneration): boolean; /** Converts a failed or interrupted saved-state startup into a durable safety hold. */ holdFailedRecovery(error: string, generation?: SandboxGeneration): void; /** Records runtime protocol support; does not itself grant lifecycle command readiness. */ @@ -270,7 +290,8 @@ export interface SandboxStorage { vnc: { url: string; password: string } | null; ttyd: { url: string | null; token: string } | null; tunnelUrls: Record | null; - } + }, + expectedProviderObjectId?: string ): Promise; /** Update sandbox Modal object ID (for snapshot API) */ updateSandboxModalObjectId(modalObjectId: string | null): void; @@ -515,6 +536,18 @@ export class SandboxLifecycleManager private isSpawningSandbox = false; private isTerminatingSandbox = false; private providerStartupPending = false; + private bridgeResolution: SandboxGeneration | null = null; + private bridgeRetryGeneration: SandboxGeneration | null = null; + private bridgeStartupClaim: SandboxGeneration | null = null; + private bridgeResolvedStartup: { + generation: SandboxGeneration; + result: ResolveSandboxResult; + } | null = null; + private vmStartupAuth: { + generation: SandboxGeneration; + sessionId: string; + token: string; + } | null = null; retireShutdownAccess(): void { this.clearSandboxAccessState(); this.wsManager.detachSandboxWebSocket(1000, "Sandbox state preserved"); @@ -551,7 +584,8 @@ export class SandboxLifecycleManager private readonly idGenerator: IdGenerator, private readonly shutdown: SandboxShutdownLifecycle, private readonly config: SandboxLifecycleConfig, - private readonly imageBuildLookup?: ImageBuildLookup + private readonly imageBuildLookup?: ImageBuildLookup, + private readonly backgroundTasks?: BackgroundTasks ) {} /** @@ -850,10 +884,15 @@ export class SandboxLifecycleManager ...multiRepoFields, }; + if (this.provider.name === "modal-vm") + this.vmStartupAuth = { generation, sessionId, token: sandboxAuthToken }; + let result: CreateSandboxResult; try { await this.recordPendingProviderReference(generation, createConfig); - result = await this.provider.createSandbox(createConfig); + const created = await this.createWithVmRecovery(createConfig, generation); + if (!created) return; + result = created; } catch (error) { if (!selectedImage) throw error; if (!(error instanceof PrebuiltImageUnavailableError)) { @@ -884,23 +923,25 @@ export class SandboxLifecycleManager const retryNow = Math.max(Date.now(), now + 1); const retry = this.spawnGeneration(session, retryNow); generation = retry; + this.vmStartupAuth = null; ({ sandboxAuthToken, expectedSandboxId } = await this.reserveSpawnIdentity(retry, { preserveProviderObjectId: false, shutdownPolicy: shutdownPolicyForLaunch("new", null), })); - await this.recordPendingProviderReference(generation, { - ...createConfig, - sandboxId: expectedSandboxId, - generationCreatedAtMs: retry.createdAt, - }); - result = await this.provider.createSandbox({ + if (this.provider.name === "modal-vm") + this.vmStartupAuth = { generation, sessionId, token: sandboxAuthToken }; + const retryConfig: CreateSandboxConfig = { ...createConfig, sandboxId: expectedSandboxId, generationCreatedAtMs: retry.createdAt, sandboxAuthToken, prebuiltImageId: null, prebuiltImageSha: null, - }); + }; + await this.recordPendingProviderReference(generation, retryConfig); + const created = await this.createWithVmRecovery(retryConfig, generation); + if (!created) return; + result = created; } if (!(await this.claimProviderStartup(generation, result.providerObjectId, result.lifetime))) @@ -975,6 +1016,8 @@ export class SandboxLifecycleManager } finally { this.isSpawningSandbox = false; this.providerStartupPending = false; + if (this.vmStartupAuth?.generation === generation && this.bridgeStartupClaim !== generation) + this.vmStartupAuth = null; } } @@ -1264,9 +1307,23 @@ export class SandboxLifecycleManager sandboxSettings, ...multiRepoSpawnFields(repositories), }; + if (this.provider.name === "modal-vm") + this.vmStartupAuth = { + generation, + sessionId: restoreConfig.sessionId, + token: sandboxAuthToken, + }; await this.recordPendingProviderReference(generation, restoreConfig); this.shutdown.markRecoveryInvoked(generation); - const result = await this.provider.restoreFromSnapshot(restoreConfig); + let result; + try { + result = await this.provider.restoreFromSnapshot(restoreConfig); + } catch (error) { + if (!this.provider.isUnknownStartupError?.(error)) throw error; + const recovered = await this.resolveUnknownVmStartup(generation, restoreConfig); + if (!recovered) return; + result = { ...recovered, success: true as const }; + } if (result.success) { if ( @@ -1355,6 +1412,8 @@ export class SandboxLifecycleManager } finally { this.isSpawningSandbox = false; this.providerStartupPending = false; + if (this.vmStartupAuth?.generation === generation && this.bridgeStartupClaim !== generation) + this.vmStartupAuth = null; } } @@ -1436,10 +1495,15 @@ export class SandboxLifecycleManager ? await this.storage.getSandboxAccessSecret("ttyd") : null; const validTtydToken = ttydToken && isJwtUnexpired(ttydToken) ? ttydToken : null; - const replaceForTerminalCredential = Boolean(result.ttydUrl && !validTtydToken); - if (replaceForTerminalCredential && restoringSavedState) { - this.shutdown.holdFailedRecovery("Terminal credential is missing or expired", generation); - return; + if (result.ttydUrl && !validTtydToken) { + // Terminal tokens are signed with the sandbox auth token, which is kept + // only as a hash, so an expired or missing one cannot be renewed. The + // resumed sandbox holds the workspace; keep it without terminal access. + this.log.warn("Terminal credential unavailable; resuming without terminal access", { + event: "sandbox.resume_terminal_credential_unavailable", + provider_object_id: finalProviderObjectId, + reason: ttydToken ? "invalid_or_expired" : "missing", + }); } let completed: boolean; try { @@ -1450,12 +1514,7 @@ export class SandboxLifecycleManager ? { url: result.codeServerUrl, password: result.codeServerPassword } : null, vnc: result.vncAccess ?? null, - ttyd: validTtydToken - ? { - url: replaceForTerminalCredential ? null : (result.ttydUrl ?? null), - token: validTtydToken, - } - : null, + ttyd: validTtydToken ? { url: result.ttydUrl ?? null, token: validTtydToken } : null, tunnelUrls: result.tunnelUrls ?? null, }); } catch (error) { @@ -1479,16 +1538,6 @@ export class SandboxLifecycleManager await this.shutdown.recordProviderStartup(generation, result.lifetime); startupClaimed = true; - if (replaceForTerminalCredential) { - this.log.info("Terminal credential unavailable; replacing resumed sandbox", { - event: "sandbox.resume_terminal_credential_unavailable", - provider_object_id: finalProviderObjectId, - reason: ttydToken ? "invalid_or_expired" : "missing", - }); - await this.doSpawn(previousGeneration); - return; - } - if (!this.broadcastSandboxDashboardUrl(finalProviderObjectId)) { this.broadcaster.broadcast({ type: "sandbox_access_changed" }); } @@ -1788,6 +1837,20 @@ export class SandboxLifecycleManager } } + /** + * Hold a failed boot of the retained source instead of destroying it. That + * sandbox is the saved workspace itself, and a fence would revoke the + * credential its next resume needs. Like a failed retained resume, it waits + * for an explicit recovery, which retires the source before resuming it + * again. Resolves false for any other generation. + */ + private holdFailedRetainedBoot(sandbox: SandboxRow, reason: string): boolean { + return this.shutdown.holdFailedRetainedBoot(reason, { + sandboxId: sandbox.modal_sandbox_id, + createdAt: sandbox.created_at, + }); + } + /** * Give up on a generation whose bridge never arrived. The row is failed and * the breaker charged before the provider stop, so a prompt landing mid-stop @@ -1805,7 +1868,11 @@ export class SandboxLifecycleManager this.storage.updateSandboxStatus("failed"); this.recordSpawnFailure(ctx.now, ctx.sandbox.created_at); this.clearSandboxAccessState(); - if (this.canStopProviderSandbox()) { + const held = this.holdFailedRetainedBoot( + ctx.sandbox, + "Sandbox failed to connect within the allowed time" + ); + if (!held && this.canStopProviderSandbox()) { // Fenced before the stop: a bridge arriving while the stop is in // flight is refused at the door instead of self-healing into a // container being killed. Where the provider cannot be stopped the @@ -1822,7 +1889,9 @@ export class SandboxLifecycleManager } this.broadcaster.broadcast({ type: "sandbox_status", status: "failed" }); this.reportSandboxError( - "Sandbox failed to connect within the allowed time. It will be retried on your next message." + held + ? "Sandbox failed to connect within the allowed time." + : "Sandbox failed to connect within the allowed time. It will be retried on your next message." ); return "sandbox_failed"; } @@ -1931,9 +2000,10 @@ export class SandboxLifecycleManager * stopped; only then is the row failed. The failure is published and * persisted before the provider stop yields, and the spawn guard is held * across it, so a prompt arriving mid-stop neither waits to learn the boot - * died nor reserves a replacement that inherits this failure. Returns the - * failure text so the alarm handler can fail the pending prompt with the - * same words. + * died nor reserves a replacement that inherits this failure. A boot of the + * retained source is held instead, and its runtime and sandbox are left for + * the recovery. Returns the failure text so the alarm handler can fail the + * pending prompt with the same words. */ private async failBootBudget(elapsedMs: number, ctx: AlarmContext): Promise { const bootPhase = parseStoredSandboxBootPhase(ctx.sandbox.boot_phase); @@ -1947,13 +2017,17 @@ export class SandboxLifecycleManager elapsed_ms: elapsedMs, timeout_ms: this.config.bootBudget.timeoutMs, }); - this.wsManager.sendToSandbox({ type: "shutdown" }); - this.storage.fenceSandboxGeneration(); + const held = this.holdFailedRetainedBoot(ctx.sandbox, reason); + if (!held) { + this.wsManager.sendToSandbox({ type: "shutdown" }); + this.storage.fenceSandboxGeneration(); + } this.storage.updateSandboxStatus("failed"); this.recordSpawnFailure(ctx.now, ctx.sandbox.created_at); this.clearSandboxAccessState(); this.broadcaster.broadcast({ type: "sandbox_status", status: "failed" }); this.reportSandboxError(reason); + if (held) return { kind: "boot_budget_exceeded", reason }; this.wsManager.detachSandboxWebSocket(1000, "Boot budget exceeded"); if (this.canStopProviderSandbox()) { this.isTerminatingSandbox = true; @@ -2074,7 +2148,8 @@ export class SandboxLifecycleManager * provider allows. Resolves true only when this call took the sandbox down, * which is the caller's cue to re-evaluate the queue. Serving executions * remain fenced by preservation until explicit recovery; only failed boots - * may automatically get a clean replacement. A row + * may automatically get a clean replacement, and a failed boot of the + * retained source is held instead. A row * that is already dead — including one the connect watchdog failed while * its boot was still running — resolves false: there is nothing to * terminate, and re-driving the queue for it would spawn a replacement for @@ -2102,11 +2177,13 @@ export class SandboxLifecycleManager return ownership === "owned"; } if (!this.isCurrentSandboxState(sandbox)) return false; + const held = this.holdFailedRetainedBoot(sandbox, reason); this.storage.updateSandboxStatus("failed"); this.recordSpawnFailure(Date.now(), sandbox.created_at); this.broadcaster.broadcast({ type: "sandbox_status", status: "failed" }); this.reportSandboxError(reason); this.clearSandboxAccessState(); + if (held) return false; const canStopProvider = this.canStopProviderSandbox(); if (!canStopProvider) this.wsManager.sendToSandbox({ type: "shutdown" }); @@ -2164,6 +2241,7 @@ export class SandboxLifecycleManager if (!row) return false; const generation = { sandboxId: row.modal_sandbox_id, createdAt: row.created_at }; if (!this.storage.markSandboxReady(generation)) return false; + this.resolvePendingBridge(generation); this.log.info("sandbox.ready", { event: "sandbox.ready", harness: harness ?? null }); this.updateLastActivity(timestamp); this.broadcaster.broadcast({ type: "sandbox_status", status: "ready" }); @@ -2380,7 +2458,18 @@ export class SandboxLifecycleManager sessionId: string, sandboxId: string ): Promise { - const token = await mintJwt( + const token = await this.mintTtydToken(sandboxAuthToken, sessionId, sandboxId); + + this.log.info("Storing ttyd info", { url }); + await this.storage.updateSandboxAccess("ttyd", url, token); + } + + private mintTtydToken( + sandboxAuthToken: string, + sessionId: string, + sandboxId: string + ): Promise { + return mintJwt( { sub: sessionId, sid: sandboxId, @@ -2389,9 +2478,6 @@ export class SandboxLifecycleManager }, sandboxAuthToken ); - - this.log.info("Storing ttyd info", { url }); - await this.storage.updateSandboxAccess("ttyd", url, token); } private async recordPendingProviderReference( @@ -2435,6 +2521,251 @@ export class SandboxLifecycleManager } } + private async createWithVmRecovery( + config: CreateSandboxConfig, + generation: SandboxGeneration + ): Promise { + try { + return await this.provider.createSandbox(config); + } catch (error) { + if (!this.provider.isUnknownStartupError?.(error)) throw error; + const recovered = await this.resolveUnknownVmStartup(generation, config); + return recovered ? { ...recovered, createdAt: generation.createdAt } : null; + } + } + + private knownBridgeStartup( + generation: SandboxGeneration, + row: SandboxRow | null + ): ResolveSandboxResult | null { + const known = this.bridgeResolvedStartup; + if ( + !known || + row?.modal_sandbox_id !== generation.sandboxId || + row.created_at !== generation.createdAt || + row.fenced || + !["spawning", "connecting", "ready"].includes(row.status) || + row.modal_object_id !== known.result.providerObjectId || + known.generation.sandboxId !== generation.sandboxId || + known.generation.createdAt !== generation.createdAt + ) + return null; + return known.result; + } + + private async resolveUnknownVmStartup( + generation: SandboxGeneration, + config: Pick< + CreateSandboxConfig, + "sessionId" | "sandboxId" | "generationCreatedAtMs" | "timeoutSeconds" + > + ): Promise { + if (!this.provider.resolveSandbox) return null; + const reference = this.provider.pendingSandboxAllocation?.(config)?.reference; + while (true) { + const row = this.storage.getSandbox(); + const bridged = this.knownBridgeStartup(generation, row); + if (bridged) return bridged; + const resolvedByBridge = + !!row?.modal_object_id && + row.modal_object_id !== reference && + parsePendingVmReference(row.modal_object_id) === null; + if ( + row?.modal_sandbox_id !== generation.sandboxId || + row.created_at !== generation.createdAt || + row.fenced || + !["spawning", "connecting", "ready"].includes(row.status) || + (row.modal_object_id !== reference && !resolvedByBridge) + ) + return null; + try { + return await this.provider.resolveSandbox({ + ...config, + generationCreatedAtMs: generation.createdAt, + }); + } catch (error) { + const detail = vmAllocationDetail(error); + if (detail === "other_generation") throw error; + if (detail !== "not_visible" && !this.provider.isUnknownStartupError?.(error)) throw error; + if (Date.now() - generation.createdAt >= PENDING_VM_REFERENCE_MATERIALIZE_BOUND_MS) { + const current = this.storage.getSandbox(); + const bridgedAfterLookup = this.knownBridgeStartup(generation, current); + if (bridgedAfterLookup) return bridgedAfterLookup; + if ( + current?.modal_sandbox_id === generation.sandboxId && + current.created_at === generation.createdAt && + !current.fenced && + current.modal_object_id && + parsePendingVmReference(current.modal_object_id) === null + ) { + const lifetime = this.provider.pendingSandboxAllocation?.(config)?.lifetime; + if (lifetime) + return { + sandboxId: config.sandboxId, + providerObjectId: current.modal_object_id, + lifetime, + }; + } + if (detail === "not_visible") + throw new SandboxProviderError( + "The VM allocation did not appear for this attempt. Please retry.", + "transient", + error instanceof Error ? error : undefined + ); + if ( + current?.modal_sandbox_id === generation.sandboxId && + current.created_at === generation.createdAt && + !current.fenced && + current.modal_object_id === reference + ) + this.bridgeStartupClaim = generation; + return null; + } + } + await new Promise((resolve) => setTimeout(resolve, VM_RESOLVE_RETRY_MS)); + } + } + + /** Bridge-triggered reconciliation is lookup-only and never holds readiness. */ + private resolvePendingBridge(generation: SandboxGeneration): void { + if (this.bridgeResolution) { + if ( + this.bridgeResolution.sandboxId !== generation.sandboxId || + this.bridgeResolution.createdAt !== generation.createdAt + ) + this.bridgeRetryGeneration = generation; + return; + } + if (!this.provider.resolveSandbox) return; + const row = this.storage.getSandbox(); + const reference = row?.modal_object_id; + const pending = reference ? parsePendingVmReference(reference) : null; + const session = this.sessionContext.getSession(); + if ( + !row || + row.fenced || + !["spawning", "connecting", "ready"].includes(row.status) || + row.created_at !== generation.createdAt || + row.modal_sandbox_id !== generation.sandboxId || + !reference || + !pending || + !session || + pending.sandboxId !== row.modal_sandbox_id || + pending.sessionId !== (session.session_name || session.id) + ) + return; + const config = { + sessionId: pending.sessionId, + sandboxId: pending.sandboxId, + generationCreatedAtMs: generation.createdAt, + timeoutSeconds: this.resolveSandboxTimeoutSeconds(this.parseSandboxSettings(session)), + }; + const retryDeadlineAtMs = Date.now() + PENDING_VM_REFERENCE_MATERIALIZE_BOUND_MS; + this.bridgeResolution = generation; + const work = () => + (async () => { + let result: ResolveSandboxResult; + while (true) { + const current = this.storage.getSandbox(); + if ( + current?.modal_sandbox_id !== generation.sandboxId || + current.created_at !== generation.createdAt || + current.fenced || + !["spawning", "connecting", "ready"].includes(current.status) || + current.modal_object_id !== reference + ) + return; + try { + result = await this.provider.resolveSandbox!(config); + break; + } catch (error) { + const detail = vmAllocationDetail(error); + if (detail !== "not_visible" && !this.provider.isUnknownStartupError?.(error)) + throw error; + if ( + Date.now() >= retryDeadlineAtMs || + (detail === "not_visible" && + Date.now() - generation.createdAt >= PENDING_VM_REFERENCE_MATERIALIZE_BOUND_MS) || + this.bridgeRetryGeneration + ) + return; + await new Promise((resolve) => setTimeout(resolve, VM_RESOLVE_RETRY_MS)); + } + } + if (!result.providerObjectId) return; + const auth = this.vmStartupAuth; + const terminalToken = + result.ttydUrl && + auth && + auth.generation.sandboxId === generation.sandboxId && + auth.generation.createdAt === generation.createdAt + ? await this.mintTtydToken(auth.token, auth.sessionId, generation.sandboxId!) + : null; + const committed = await this.storage.completeProviderResume( + generation, + { + providerObjectId: result.providerObjectId, + codeServer: + result.codeServerUrl && result.codeServerPassword + ? { url: result.codeServerUrl, password: result.codeServerPassword } + : null, + vnc: result.vncAccess ?? null, + ttyd: + result.ttydUrl && terminalToken + ? { url: result.ttydUrl, token: terminalToken } + : null, + tunnelUrls: result.tunnelUrls ?? null, + }, + reference + ); + if (!committed) return; + this.bridgeResolvedStartup = { + generation, + result: { + sandboxId: result.sandboxId, + providerObjectId: result.providerObjectId, + lifetime: result.lifetime, + }, + }; + if ( + this.bridgeStartupClaim?.sandboxId === generation.sandboxId && + this.bridgeStartupClaim.createdAt === generation.createdAt + ) { + this.bridgeStartupClaim = null; + try { + await this.claimProviderStartup(generation, result.providerObjectId, result.lifetime); + } finally { + if ( + this.vmStartupAuth?.generation.sandboxId === generation.sandboxId && + this.vmStartupAuth.generation.createdAt === generation.createdAt + ) + this.vmStartupAuth = null; + } + } else { + this.shutdown.recordResolvedProviderHandle?.( + generation, + reference, + result.providerObjectId + ); + } + this.broadcastProviderAccessIfConnected(); + })() + .catch((error) => { + this.log.warn("Bridge VM resolution failed", { + event: "sandbox.vm_resolve_failed", + error: error instanceof Error ? error.message : String(error), + }); + }) + .finally(() => { + this.bridgeResolution = null; + const queued = this.bridgeRetryGeneration; + this.bridgeRetryGeneration = null; + if (queued) this.resolvePendingBridge(queued); + }); + if (this.backgroundTasks) this.backgroundTasks.submit(work, { name: "sandbox.vm_resolve" }); + else void work(); + } + private async handleRejectedStartupAllocation( error: unknown, generation: SandboxGeneration | null @@ -2605,6 +2936,7 @@ export class SandboxLifecycleManager * in flight, and for the user, who sees the boot begin. */ onSandboxSocketAttached(generation: SandboxGeneration): void { + this.resolvePendingBridge(generation); if (this.storage.transitionSandboxStatus(generation, "spawning", "connecting")) { this.broadcaster.broadcast({ type: "sandbox_status", status: "connecting" }); return; diff --git a/packages/control-plane/src/sandbox/lifecycle/test-helpers.ts b/packages/control-plane/src/sandbox/lifecycle/test-helpers.ts index 7e5065989d..4094ac152c 100644 --- a/packages/control-plane/src/sandbox/lifecycle/test-helpers.ts +++ b/packages/control-plane/src/sandbox/lifecycle/test-helpers.ts @@ -468,6 +468,7 @@ export function createUnmanagedShutdown() { startupDecision: vi.fn(() => ({ kind: "normal", })), + holdFailedRetainedBoot: vi.fn(() => false), holdFailedRecovery: vi.fn(), runtimeReady: vi.fn(), generationReady: vi.fn(), diff --git a/packages/control-plane/src/sandbox/lifecycle/vm-resolve.test.ts b/packages/control-plane/src/sandbox/lifecycle/vm-resolve.test.ts new file mode 100644 index 0000000000..b972627940 --- /dev/null +++ b/packages/control-plane/src/sandbox/lifecycle/vm-resolve.test.ts @@ -0,0 +1,560 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { ModalApiError, type ModalClient, type ResolveVmSandboxResponse } from "../client"; +import { ModalSandboxProvider } from "../providers/modal-provider"; +import { formatPendingVmReference } from "../providers/pending-vm-reference"; +import { RequestDeadlineError } from "../request-deadline"; +import { SandboxLifecycleManager } from "./manager"; +import type { ImageBuildLookup } from "./image-selection"; +import { computeRepositoriesFingerprint } from "../../image-builds/fingerprint"; +import { PENDING_VM_REFERENCE_MATERIALIZE_BOUND_MS } from "./decisions"; +import { SandboxShutdownCoordinator } from "../../session/sandbox-shutdown"; +import type { ShutdownRecord } from "../../session/sandbox-shutdown-repository"; +import { + createMockSandbox, + createMockSession, + createMockStorage, + createMockBroadcaster, + createMockWebSocketManager, + createMockAlarmScheduler, + createMockIdGenerator, + createTestConfig, +} from "./test-helpers"; +import { COMPATIBLE_RUNTIME_VERSION } from "../../image-builds/test-helpers"; + +function fixture(action: "create" | "restore" = "create", imageBuildLookup?: ImageBuildLookup) { + const sandbox = createMockSandbox({ + status: action === "create" ? "pending" : "stopped", + snapshot_image_id: action === "restore" ? "im-saved" : null, + snapshot_runtime_version: action === "restore" ? COMPATIBLE_RUNTIME_VERSION : null, + }); + const session = createMockSession({ + code_server_enabled: 1, + vnc_enabled: 1, + sandbox_settings: JSON.stringify({ sandboxTimeoutMs: 3_600_000 }), + }); + const storage = createMockStorage( + session, + sandbox, + undefined, + imageBuildLookup ? [{ repoOwner: "testowner", repoName: "testrepo", baseBranch: "main" }] : [] + ); + const broadcaster = createMockBroadcaster(); + const wsManager = createMockWebSocketManager(); + const providerResponse: ResolveVmSandboxResponse = { + sandboxId: "unused", + modalObjectId: "sb-real", + sandboxBackend: "modal-vm", + codeServerUrl: "https://editor.example", + codeServerPassword: "editor-password", + vncUrl: "https://desktop.example", + vncPassword: "desktop-password", + ttydUrl: "https://terminal.example", + tunnelUrls: { "8080": "https://port.example" }, + }; + const client = { + createSandbox: vi.fn( + async ( + _config: unknown + ): Promise<{ + sandboxId: string; + modalObjectId: string; + sandboxBackend: string; + createdAt: number; + }> => { + throw new RequestDeadlineError("Modal", "createSandbox", 60_000); + } + ), + restoreSandbox: vi.fn(async () => { + throw new ModalApiError("pending race", 409, "race_pending"); + }), + resolveVmSandbox: vi.fn( + async (req: { sandboxId: string }): Promise => ({ + ...providerResponse, + sandboxId: req.sandboxId, + }) + ), + stopSandbox: vi.fn(async () => {}), + }; + const provider = new ModalSandboxProvider(client as unknown as ModalClient, "modal-vm"); + const backgroundTasks = { submit: vi.fn((task: () => Promise) => void task()) }; + let state: ShutdownRecord | null = null; + const store = { + read: () => (state ? structuredClone(state) : null), + write: (next: ShutdownRecord) => { + state = structuredClone(next); + }, + }; + if (action === "restore") { + store.write({ + phase: "saved", + generation: { sandboxId: sandbox.modal_sandbox_id!, createdAt: sandbox.created_at }, + provider: "modal-vm", + providerObjectId: null, + sourceRetired: true, + lifetimeKind: "none", + expiresAtMs: null, + drainAtMs: null, + generationReady: true, + lifecyclePolicy: "confirmed", + receipt: { + kind: "snapshot", + artifactId: "im-saved", + provider: "modal-vm", + savedAtMs: Date.now(), + runtimeVersion: COMPATIBLE_RUNTIME_VERSION, + }, + }); + } + const deps = { + store, + provider, + sandbox: storage, + session: { getSession: () => session, transaction: (fn: () => T) => fn() }, + messages: { getProcessingMessage: () => null }, + failures: { record: vi.fn(), deliver: vi.fn() }, + messenger: broadcaster, + sockets: { getSandboxSocket: () => null }, + alarm: createMockAlarmScheduler(), + background: { submit: vi.fn() }, + onLifecycleChange: vi.fn(async () => {}), + reconcileStatusFromMessages: vi.fn(async () => {}), + retireAccess: vi.fn(), + }; + const makeManager = () => + new SandboxLifecycleManager( + provider, + storage, + storage, + broadcaster, + wsManager, + createMockAlarmScheduler(), + createMockIdGenerator(), + new SandboxShutdownCoordinator(deps as never), + createTestConfig(), + imageBuildLookup, + backgroundTasks + ); + return { sandbox, storage, broadcaster, client, store, makeManager, wsManager, backgroundTasks }; +} + +describe("modal-vm startup resolution", () => { + afterEach(() => vi.useRealTimers()); + + it.each(["create", "restore"] as const)( + "recovers an unknown %s without resetting lifetime", + async (action) => { + vi.useFakeTimers({ toFake: ["Date"] }); + vi.setSystemTime(new Date("2030-01-01T00:00:00Z")); + const f = fixture(action); + const manager = f.makeManager(); + await manager.spawnSandbox(); + expect(f.sandbox.modal_object_id).toBe("sb-real"); + expect(f.sandbox.status).toBe("connecting"); + expect(f.store.read()).toMatchObject({ + providerObjectId: "sb-real", + expiresAtMs: f.sandbox.created_at + 3_600_000, + }); + expect(f.sandbox.code_server_url).toBe("https://editor.example"); + expect(f.sandbox.vnc_password).toBe("desktop-password"); + expect(f.sandbox.tunnel_urls).toBe(JSON.stringify({ "8080": "https://port.example" })); + expect(f.sandbox.ttyd_token).toBeTruthy(); + expect(f.broadcaster.messages).not.toContainEqual( + expect.objectContaining({ type: "sandbox_error" }) + ); + expect(f.sandbox.spawn_failure_count).toBe(0); + } + ); + + it("fails once when the allocation remains invisible past the bound, allowing respawn", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2030-01-01T00:00:00Z")); + const f = fixture(); + f.client.resolveVmSandbox.mockRejectedValue(new ModalApiError("invisible", 409, "not_visible")); + const manager = f.makeManager(); + const spawning = manager.spawnSandbox(); + await vi.waitFor(() => expect(f.client.resolveVmSandbox).toHaveBeenCalledOnce()); + expect(f.sandbox.status).toBe("spawning"); + await vi.advanceTimersByTimeAsync(PENDING_VM_REFERENCE_MATERIALIZE_BOUND_MS + 16_000); + await spawning; + expect(f.sandbox.status).toBe("failed"); + expect(f.broadcaster.messages).toContainEqual( + expect.objectContaining({ type: "sandbox_error" }) + ); + expect(f.storage.transitionSandboxStatus).toHaveBeenCalledTimes(1); + f.client.createSandbox.mockResolvedValueOnce({ + sandboxId: "next", + modalObjectId: "sb-next", + sandboxBackend: "modal-vm", + createdAt: Date.now(), + }); + await manager.spawnSandbox(); + expect(f.client.createSandbox).toHaveBeenCalledTimes(2); + }); + + it("fails definitively for another generation", async () => { + const f = fixture(); + f.client.resolveVmSandbox.mockRejectedValue( + new ModalApiError("occupied", 409, "other_generation") + ); + await f.makeManager().spawnSandbox(); + expect(f.sandbox.status).toBe("failed"); + expect(f.client.resolveVmSandbox).toHaveBeenCalledOnce(); + }); + + it("resolves an ambiguous base-image retry after a prebuilt image is unavailable", async () => { + const imageBuildLookup: ImageBuildLookup = { + getLatestReady: vi.fn(async () => ({ + id: "image-build-1", + provider_image_id: "im-prebuilt", + repositories_fingerprint: await computeRepositoriesFingerprint([ + { repoOwner: "testowner", repoName: "testrepo", baseBranch: "main" }, + ]), + repository_shas: JSON.stringify([ + { repoOwner: "testowner", repoName: "testrepo", baseSha: "sha-1" }, + ]), + runtime_version: COMPATIBLE_RUNTIME_VERSION, + })), + markRestoreFailed: vi.fn(async () => true), + }; + const f = fixture("create", imageBuildLookup); + f.client.createSandbox + .mockRejectedValueOnce(new ModalApiError("prebuilt unavailable", 410)) + .mockRejectedValueOnce(new RequestDeadlineError("Modal", "createSandbox", 60_000)); + await f.makeManager().spawnSandbox(); + expect(f.client.createSandbox).toHaveBeenCalledTimes(2); + expect(f.client.createSandbox.mock.calls[1][0]).toMatchObject({ prebuiltImageId: null }); + expect(imageBuildLookup.markRestoreFailed).toHaveBeenCalledOnce(); + expect(f.client.resolveVmSandbox).toHaveBeenCalledOnce(); + expect(f.sandbox.modal_object_id).toBe("sb-real"); + }); + + it("bounds repeated transient lookup errors without failing the pending generation", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2030-01-01T00:00:00Z")); + const f = fixture(); + f.client.resolveVmSandbox.mockRejectedValue(new ModalApiError("unavailable", 503)); + const manager = f.makeManager(); + const spawning = manager.spawnSandbox(); + await vi.waitFor(() => expect(f.client.resolveVmSandbox).toHaveBeenCalledOnce()); + await vi.advanceTimersByTimeAsync(PENDING_VM_REFERENCE_MATERIALIZE_BOUND_MS + 20_000); + await spawning; + expect(manager.isProviderStartupPending()).toBe(false); + expect(f.sandbox.status).toBe("spawning"); + expect(f.sandbox.modal_object_id).toBe( + formatPendingVmReference("test-session", f.sandbox.modal_sandbox_id!) + ); + expect(f.broadcaster.messages).not.toContainEqual( + expect.objectContaining({ type: "sandbox_error" }) + ); + }); + + it("completes a restore after bounded transient errors when its bridge later resolves", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2030-01-01T00:00:00Z")); + const f = fixture("restore"); + const manager = f.makeManager(); + f.client.resolveVmSandbox.mockRejectedValue(new ModalApiError("unavailable", 503)); + const restoring = manager.spawnSandbox(); + await vi.waitFor(() => expect(f.client.resolveVmSandbox).toHaveBeenCalledOnce()); + await vi.advanceTimersByTimeAsync(PENDING_VM_REFERENCE_MATERIALIZE_BOUND_MS + 10_000); + await restoring; + expect(f.store.read()).toMatchObject({ phase: "restoring", restoreInvoked: true }); + f.client.resolveVmSandbox.mockResolvedValue({ + sandboxId: f.sandbox.modal_sandbox_id!, + modalObjectId: "sb-real", + sandboxBackend: "modal-vm", + ttydUrl: "https://terminal.example", + }); + const generation = { sandboxId: f.sandbox.modal_sandbox_id!, createdAt: f.sandbox.created_at }; + manager.onSandboxSocketAttached(generation); + await vi.waitFor(() => expect(f.store.read()?.phase).toBe("running")); + expect(manager.onRuntimeReady(Date.now(), undefined, 1)).toBe(true); + manager.onShutdownGenerationReady({ + type: "sandbox_generation_ready", + generation, + sandboxId: generation.sandboxId, + timestamp: Date.now(), + }); + expect(f.sandbox.modal_object_id).toBe("sb-real"); + expect(f.sandbox.ttyd_token).toBeTruthy(); + expect(manager.pushAdmissionDecision()).toBe("ready"); + }); + + it("claims a bridge-resolved restore without a second network lookup", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2030-01-01T00:00:00Z")); + const f = fixture("restore"); + let rejectRestore!: (error: Error) => void; + f.client.restoreSandbox.mockImplementationOnce( + () => new Promise((_resolve, reject) => (rejectRestore = reject)) + ); + const manager = f.makeManager(); + const restoring = manager.spawnSandbox(); + await vi.waitFor(() => expect(f.client.restoreSandbox).toHaveBeenCalledOnce()); + const generation = { sandboxId: f.sandbox.modal_sandbox_id!, createdAt: f.sandbox.created_at }; + manager.onSandboxSocketAttached(generation); + await vi.waitFor(() => expect(f.sandbox.modal_object_id).toBe("sb-real")); + rejectRestore(new ModalApiError("pending race", 409, "race_pending")); + await restoring; + expect(f.client.resolveVmSandbox).toHaveBeenCalledOnce(); + expect(f.store.read()).toMatchObject({ phase: "running", providerObjectId: "sb-real" }); + expect(f.store.read()?.restoreInvoked).toBeUndefined(); + expect(f.sandbox.ttyd_token).toBeTruthy(); + }); + + it("claims a bridge result committed during the final inconclusive lookup", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2030-01-01T00:00:00Z")); + const f = fixture("restore"); + let rejectLookup!: (error: Error) => void; + f.client.resolveVmSandbox.mockImplementationOnce( + () => new Promise((_resolve, reject) => (rejectLookup = reject)) + ); + const manager = f.makeManager(); + const restoring = manager.spawnSandbox(); + await vi.waitFor(() => expect(f.client.resolveVmSandbox).toHaveBeenCalledOnce()); + const generation = { sandboxId: f.sandbox.modal_sandbox_id!, createdAt: f.sandbox.created_at }; + f.sandbox.status = "connecting"; + const pending = f.sandbox.modal_object_id!; + f.sandbox.modal_object_id = "sb-real"; + f.store.write({ ...f.store.read()!, providerObjectId: "sb-real" }); + vi.setSystemTime(generation.createdAt + PENDING_VM_REFERENCE_MATERIALIZE_BOUND_MS); + rejectLookup(new ModalApiError("invisible", 409, "not_visible")); + await restoring; + expect(f.sandbox.modal_object_id).toBe("sb-real"); + expect(f.store.read()?.phase).toBe("running"); + expect(f.store.read()?.restoreInvoked).toBeUndefined(); + expect(pending).toBe(formatPendingVmReference("test-session", generation.sandboxId)); + }); + + it("stops resolving when the row is fenced mid-loop", async () => { + vi.useFakeTimers(); + const f = fixture(); + f.client.resolveVmSandbox.mockRejectedValue(new ModalApiError("invisible", 409, "not_visible")); + const spawning = f.makeManager().spawnSandbox(); + await vi.waitFor(() => expect(f.client.resolveVmSandbox).toHaveBeenCalledOnce()); + f.sandbox.status = "failed"; + f.sandbox.fenced = 1; + await vi.advanceTimersByTimeAsync(20_000); + await spawning; + expect(f.client.resolveVmSandbox).toHaveBeenCalledOnce(); + expect(f.sandbox.modal_object_id).not.toBe("sb-real"); + }); + + it("destroys a result that arrives after the row is fenced", async () => { + const f = fixture(); + let resolve!: (value: ResolveVmSandboxResponse) => void; + f.client.resolveVmSandbox.mockImplementationOnce(() => new Promise((done) => (resolve = done))); + const spawning = f.makeManager().spawnSandbox(); + await vi.waitFor(() => expect(f.client.resolveVmSandbox).toHaveBeenCalledOnce()); + f.sandbox.status = "failed"; + f.sandbox.fenced = 1; + resolve({ + sandboxId: f.sandbox.modal_sandbox_id!, + modalObjectId: "sb-late", + sandboxBackend: "modal-vm", + }); + await spawning; + expect(f.client.stopSandbox).toHaveBeenCalledWith( + expect.objectContaining({ providerObjectId: "sb-late" }), + undefined + ); + expect(f.sandbox.modal_object_id).not.toBe("sb-late"); + }); + + it("reconciles a restarted bridge without blocking readiness or writing a replaced generation", async () => { + const f = fixture(); + const original = f.makeManager(); + f.client.createSandbox.mockImplementationOnce(() => new Promise(() => {})); + void original.spawnSandbox(); + await vi.waitFor(() => expect(f.client.createSandbox).toHaveBeenCalledOnce()); + const generation = { sandboxId: f.sandbox.modal_sandbox_id!, createdAt: f.sandbox.created_at }; + expect(f.sandbox.modal_object_id).toBe( + formatPendingVmReference("test-session", generation.sandboxId) + ); + let resolve!: (value: Awaited>) => void; + f.client.resolveVmSandbox.mockImplementationOnce(() => new Promise((done) => (resolve = done))); + const restarted = f.makeManager(); + restarted.onSandboxSocketAttached(generation); + expect(f.backgroundTasks.submit).toHaveBeenCalledWith(expect.any(Function), { + name: "sandbox.vm_resolve", + }); + expect(restarted.onRuntimeReady(Date.now(), undefined, 1)).toBe(true); + await vi.waitFor(() => expect(f.client.resolveVmSandbox).toHaveBeenCalledOnce()); + expect(f.sandbox.modal_object_id).not.toBe("sb-real"); + expect(f.client.resolveVmSandbox).toHaveBeenCalledOnce(); + resolve({ + sandboxId: generation.sandboxId, + modalObjectId: "sb-real", + sandboxBackend: "modal-vm", + codeServerUrl: "https://editor.example", + codeServerPassword: "password", + }); + await vi.waitFor(() => expect(f.sandbox.modal_object_id).toBe("sb-real")); + expect(f.store.read()?.providerObjectId).toBe("sb-real"); + expect(f.store.read()?.expiresAtMs).toBe(generation.createdAt + 3_600_000); + expect(f.sandbox.ttyd_token).toBeNull(); + }); + + it("mints terminal access when the bridge resolves while the original instance holds the token", async () => { + const f = fixture(); + f.client.createSandbox.mockImplementationOnce(() => new Promise(() => {})); + const manager = f.makeManager(); + void manager.spawnSandbox(); + await vi.waitFor(() => expect(f.client.createSandbox).toHaveBeenCalledOnce()); + manager.onSandboxSocketAttached({ + sandboxId: f.sandbox.modal_sandbox_id!, + createdAt: f.sandbox.created_at, + }); + await vi.waitFor(() => expect(f.sandbox.modal_object_id).toBe("sb-real")); + expect(f.sandbox.ttyd_url).toBe("https://terminal.example"); + expect(f.sandbox.ttyd_token).toBeTruthy(); + }); + + it("claims a restore when the bridge resolved its handle before the lost response", async () => { + const f = fixture("restore"); + let rejectRestore!: (error: Error) => void; + f.client.restoreSandbox.mockImplementationOnce( + () => new Promise((_resolve, reject) => (rejectRestore = reject)) + ); + const manager = f.makeManager(); + const restoring = manager.spawnSandbox(); + await vi.waitFor(() => expect(f.client.restoreSandbox).toHaveBeenCalledOnce()); + const generation = { sandboxId: f.sandbox.modal_sandbox_id!, createdAt: f.sandbox.created_at }; + manager.onSandboxSocketAttached(generation); + await vi.waitFor(() => expect(f.sandbox.modal_object_id).toBe("sb-real")); + expect(f.store.read()).toMatchObject({ phase: "restoring", restoreInvoked: true }); + + expect(manager.onRuntimeReady(Date.now(), undefined, 1)).toBe(true); + f.sandbox.status = "ready"; + manager.onShutdownGenerationReady({ + type: "sandbox_generation_ready", + generation, + sandboxId: generation.sandboxId, + timestamp: Date.now(), + }); + rejectRestore(new ModalApiError("pending race", 409, "race_pending")); + await restoring; + + expect(f.client.resolveVmSandbox).toHaveBeenCalledOnce(); + expect(f.store.read()).toMatchObject({ phase: "running", providerObjectId: "sb-real" }); + expect(f.store.read()?.restoreInvoked).toBeUndefined(); + expect(manager.pushAdmissionDecision()).toBe("ready"); + expect(f.sandbox.modal_object_id).toBe("sb-real"); + }); + + it("does not attach bridge access to a newer generation", async () => { + const f = fixture(); + f.client.createSandbox.mockImplementationOnce(() => new Promise(() => {})); + void f.makeManager().spawnSandbox(); + await vi.waitFor(() => expect(f.client.createSandbox).toHaveBeenCalledOnce()); + let resolve!: (value: ResolveVmSandboxResponse) => void; + f.client.resolveVmSandbox.mockImplementationOnce(() => new Promise((done) => (resolve = done))); + const restarted = f.makeManager(); + restarted.onSandboxSocketAttached({ + sandboxId: f.sandbox.modal_sandbox_id!, + createdAt: f.sandbox.created_at, + }); + await vi.waitFor(() => expect(f.client.resolveVmSandbox).toHaveBeenCalledOnce()); + const oldSandboxId = f.sandbox.modal_sandbox_id!; + f.sandbox.modal_sandbox_id = "newer-generation"; + f.sandbox.created_at += 1; + f.sandbox.modal_object_id = "sb-newer"; + resolve({ sandboxId: oldSandboxId, modalObjectId: "sb-real", sandboxBackend: "modal-vm" }); + await vi.waitFor(() => expect(f.storage.completeProviderResume).toHaveBeenCalledOnce()); + expect(f.sandbox.modal_object_id).toBe("sb-newer"); + expect(f.store.read()?.providerObjectId).not.toBe("sb-real"); + }); + + it("retries transient bridge lookups after readiness until the same generation resolves", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2030-01-01T00:00:00Z")); + const f = fixture(); + f.client.createSandbox.mockImplementationOnce(() => new Promise(() => {})); + void f.makeManager().spawnSandbox(); + await vi.waitFor(() => expect(f.client.createSandbox).toHaveBeenCalledOnce()); + f.client.resolveVmSandbox + .mockRejectedValueOnce(new ModalApiError("invisible", 409, "not_visible")) + .mockRejectedValueOnce(new ModalApiError("unavailable", 503)); + const restarted = f.makeManager(); + const generation = { sandboxId: f.sandbox.modal_sandbox_id!, createdAt: f.sandbox.created_at }; + restarted.onSandboxSocketAttached(generation); + expect(restarted.onRuntimeReady(Date.now(), undefined, 1)).toBe(true); + await vi.advanceTimersByTimeAsync(25_000); + expect(f.client.resolveVmSandbox).toHaveBeenCalledTimes(3); + expect(f.sandbox.modal_object_id).toBe("sb-real"); + }); + + it("retries a transient bridge error even when the generation predates the visibility bound", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2030-01-01T00:00:00Z")); + const f = fixture(); + f.sandbox.status = "ready"; + f.sandbox.created_at = Date.now() - PENDING_VM_REFERENCE_MATERIALIZE_BOUND_MS - 1; + f.sandbox.modal_object_id = formatPendingVmReference( + "test-session", + f.sandbox.modal_sandbox_id! + ); + f.client.resolveVmSandbox.mockRejectedValueOnce(new ModalApiError("unavailable", 503)); + f.makeManager().onSandboxSocketAttached({ + sandboxId: f.sandbox.modal_sandbox_id!, + createdAt: f.sandbox.created_at, + }); + await vi.advanceTimersByTimeAsync(15_000); + expect(f.client.resolveVmSandbox).toHaveBeenCalledTimes(2); + expect(f.sandbox.modal_object_id).toBe("sb-real"); + }); + + it("stops bridge retries after the bounded window while leaving the pending handle intact", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2030-01-01T00:00:00Z")); + const f = fixture(); + f.client.createSandbox.mockImplementationOnce(() => new Promise(() => {})); + void f.makeManager().spawnSandbox(); + await vi.waitFor(() => expect(f.client.createSandbox).toHaveBeenCalledOnce()); + f.client.resolveVmSandbox.mockRejectedValue(new ModalApiError("unavailable", 503)); + const restarted = f.makeManager(); + restarted.onSandboxSocketAttached({ + sandboxId: f.sandbox.modal_sandbox_id!, + createdAt: f.sandbox.created_at, + }); + expect(restarted.onRuntimeReady(Date.now(), undefined, 1)).toBe(true); + await vi.advanceTimersByTimeAsync(PENDING_VM_REFERENCE_MATERIALIZE_BOUND_MS + 20_000); + const attempts = f.client.resolveVmSandbox.mock.calls.length; + await vi.advanceTimersByTimeAsync(30_000); + expect(f.client.resolveVmSandbox).toHaveBeenCalledTimes(attempts); + expect(f.sandbox.modal_object_id).toBe( + formatPendingVmReference("test-session", f.sandbox.modal_sandbox_id!) + ); + }); + + it("retries the newer generation if its bridge attaches during an older lookup", async () => { + const f = fixture(); + f.client.createSandbox.mockImplementationOnce(() => new Promise(() => {})); + void f.makeManager().spawnSandbox(); + await vi.waitFor(() => expect(f.client.createSandbox).toHaveBeenCalledOnce()); + const restarted = f.makeManager(); + let rejectFirst!: (error: Error) => void; + f.client.resolveVmSandbox.mockImplementationOnce( + () => new Promise((_resolve, reject) => (rejectFirst = reject)) + ); + restarted.onSandboxSocketAttached({ + sandboxId: f.sandbox.modal_sandbox_id!, + createdAt: f.sandbox.created_at, + }); + await vi.waitFor(() => expect(f.client.resolveVmSandbox).toHaveBeenCalledOnce()); + const newer = { sandboxId: "newer-generation", createdAt: f.sandbox.created_at + 1 }; + f.sandbox.modal_sandbox_id = newer.sandboxId; + f.sandbox.created_at = newer.createdAt; + f.sandbox.modal_object_id = formatPendingVmReference("test-session", newer.sandboxId); + f.sandbox.status = "connecting"; + restarted.onSandboxSocketAttached(newer); + restarted.onRuntimeReady(Date.now(), undefined, 1); + rejectFirst(new ModalApiError("invisible", 409, "not_visible")); + await vi.waitFor(() => expect(f.client.resolveVmSandbox).toHaveBeenCalledTimes(2)); + expect(f.client.resolveVmSandbox.mock.calls[1][0]).toMatchObject({ + sandboxId: newer.sandboxId, + }); + await vi.waitFor(() => expect(f.sandbox.modal_object_id).toBe("sb-real")); + }); +}); diff --git a/packages/control-plane/src/sandbox/provider.ts b/packages/control-plane/src/sandbox/provider.ts index 9c9e78d447..b4f019a90e 100644 --- a/packages/control-plane/src/sandbox/provider.ts +++ b/packages/control-plane/src/sandbox/provider.ts @@ -202,6 +202,15 @@ export interface CreateSandboxResult { tunnelUrls?: Record; } +export interface ResolveSandboxConfig { + sessionId: string; + sandboxId: string; + generationCreatedAtMs: number; + timeoutSeconds?: number; +} + +export type ResolveSandboxResult = Omit; + /** * Configuration for restoring a sandbox from a snapshot. */ @@ -587,6 +596,12 @@ export interface SandboxProvider { > ): PendingSandboxAllocation | undefined; + /** Lookup only, for a VM launch whose response was lost. */ + resolveSandbox?(config: ResolveSandboxConfig): Promise; + + /** Whether a failed launch could still have created this generation. */ + isUnknownStartupError?(error: unknown): boolean; + /** * Create a new sandbox. * diff --git a/packages/control-plane/src/sandbox/providers/modal-provider.test.ts b/packages/control-plane/src/sandbox/providers/modal-provider.test.ts index 1cf2b77f55..83ec94cb20 100644 --- a/packages/control-plane/src/sandbox/providers/modal-provider.test.ts +++ b/packages/control-plane/src/sandbox/providers/modal-provider.test.ts @@ -94,6 +94,39 @@ const testConfig = { // ==================== Tests ==================== describe("ModalSandboxProvider", () => { + it.each([ + [409, "race_pending", true], + [409, "other_generation", false], + [409, "window_closed", false], + [502, undefined, true], + [500, undefined, true], + ] as const)( + "classifies VM launch HTTP %s / %s without matching messages", + async (status, detail, unknown) => { + const error = new ModalApiError("arbitrary message", status, detail); + const provider = new ModalSandboxProvider( + createMockModalClient({ + createSandbox: async () => { + throw error; + }, + }), + "modal-vm" + ); + let caught: unknown; + try { + await provider.createSandbox({ ...testConfig, generationCreatedAtMs: Date.now() }); + } catch (cause) { + caught = cause; + } + expect(caught).toBeInstanceOf(SandboxProviderError); + expect((caught as SandboxProviderError).cause).toBe(error); + expect(provider.isUnknownStartupError(caught)).toBe(unknown); + expect((caught as SandboxProviderError).errorType).toBe( + detail === "other_generation" ? "permanent" : "transient" + ); + } + ); + it("derives a pending VM reference and lifetime from the launch reservation", async () => { vi.useFakeTimers({ toFake: ["Date"] }); try { diff --git a/packages/control-plane/src/sandbox/providers/modal-provider.ts b/packages/control-plane/src/sandbox/providers/modal-provider.ts index 5e203e8ffe..957a1902f6 100644 --- a/packages/control-plane/src/sandbox/providers/modal-provider.ts +++ b/packages/control-plane/src/sandbox/providers/modal-provider.ts @@ -5,7 +5,7 @@ * enabling unit testing and future provider abstraction. */ -import { ModalApiError } from "../client"; +import { ModalApiError, ModalVmStartupError } from "../client"; import { formatPendingVmReference, parsePendingVmReference } from "./pending-vm-reference"; import { PENDING_VM_REFERENCE_LAUNCH_WINDOW_MS, @@ -31,6 +31,8 @@ import { type CreateSandboxResult, type RestoreConfig, type RestoreResult, + type ResolveSandboxConfig, + type ResolveSandboxResult, type SnapshotConfig, type SnapshotResult, type StopConfig, @@ -115,6 +117,45 @@ export class ModalSandboxProvider implements SandboxProvider, ModalImageBuildPro }; } + isUnknownStartupError(error: unknown): boolean { + if (this.name !== "modal-vm") return false; + const cause = error instanceof SandboxProviderError ? error.cause : error; + if (cause instanceof ModalVmStartupError) + return cause.outcome === "unknown" || cause.outcome === "race_pending"; + if (cause instanceof ModalApiError) + return cause.detail === "race_pending" || cause.status >= 500; + return cause instanceof TypeError || SandboxProviderError.isTransientNetworkError(cause); + } + + async resolveSandbox(config: ResolveSandboxConfig): Promise { + if (this.name !== "modal-vm") + throw new SandboxProviderError("VM resolution requires modal-vm", "permanent"); + try { + const result = await this.client.resolveVmSandbox({ + sessionId: config.sessionId, + sandboxId: config.sandboxId, + }); + this.confirmSessionLaunch(result); + if (result.sandboxId !== config.sandboxId || !result.modalObjectId) + throw new SandboxProviderError( + "Modal VM resolution returned a different generation", + "permanent" + ); + return { + sandboxId: result.sandboxId, + providerObjectId: result.modalObjectId, + lifetime: this.launchLifetime(config), + codeServerUrl: result.codeServerUrl, + codeServerPassword: result.codeServerPassword, + vncAccess: createVncAccess(result.vncUrl, result.vncPassword), + ttydUrl: result.ttydUrl, + tunnelUrls: result.tunnelUrls, + }; + } catch (error) { + throw this.classifyError("Failed to resolve Modal VM", error); + } + } + private launchLifetime( config: Pick, observedAtMs?: number @@ -262,15 +303,6 @@ export class ModalSandboxProvider implements SandboxProvider, ModalImageBuildPro tunnelUrls: result.tunnelUrls, }; } catch (error) { - if (error instanceof ModalApiError) { - throw this.classifyErrorWithStatus( - `Restore failed with HTTP ${error.status}`, - error.status - ); - } - if (error instanceof SandboxProviderError) { - throw error; - } throw this.classifyError("Failed to restore sandbox from snapshot", error); } } @@ -520,6 +552,30 @@ export class ModalSandboxProvider implements SandboxProvider, ModalImageBuildPro */ private classifyError(message: string, error: unknown): SandboxProviderError { if (error instanceof SandboxProviderError) return error; + if (error instanceof ModalVmStartupError) + return new SandboxProviderError( + `${message}: ${error.message}`, + error.outcome === "other_generation" ? "permanent" : "transient", + error + ); + if (error instanceof ModalApiError) { + const context = `${message} with HTTP ${error.status}`; + if (this.name === "modal-vm") { + if ( + error.detail === "not_visible" || + error.detail === "window_closed" || + error.detail === "race_pending" || + error.detail === "other_generation" + ) + return new SandboxProviderError( + context, + error.detail === "other_generation" ? "permanent" : "transient", + error + ); + if (error.status >= 500) return new SandboxProviderError(context, "transient", error); + } + return this.classifyErrorWithStatus(context, error.status, error); + } if (SandboxProviderError.isTransientNetworkError(error)) { return new SandboxProviderError( `${message}: ${error instanceof Error ? error.message : String(error)}`, diff --git a/packages/control-plane/src/session/components.ts b/packages/control-plane/src/session/components.ts index 1eb6a2ae9e..2df0e0178c 100644 --- a/packages/control-plane/src/session/components.ts +++ b/packages/control-plane/src/session/components.ts @@ -59,6 +59,7 @@ import { requireRepoSecretsEncryptionKey, requireTokenEncryptionKey } from "../e import type { Env, ClientInfo } from "../types"; import type { SessionRow } from "./types"; import type { SqlDatabase } from "../db/sql-database"; +import type { BackgroundTasks } from "../platform-ports"; import type { SessionPlatform } from "./platform"; import { SessionCoreRepository } from "./session-core-repository"; // The composition root grants each consumer only its declared sandbox port. @@ -469,6 +470,7 @@ export function createSessionRuntime(platform: SessionPlatform, env: Env): Sessi wsManager, alarmScheduler, sandboxDashboardSettings, + backgroundTasks, recordWarning: (message, eventId) => recordSessionWarning(eventRepository, messenger, message, eventId), }); @@ -550,7 +552,8 @@ export function createSessionRuntime(platform: SessionPlatform, env: Env): Sessi messenger, updateLastActivity, budgetService, - usageRepository + usageRepository, + (messageId) => statusService.refreshMetricsAfterStep(messageId) ); const artifactEventHandler = new SandboxArtifactEventHandler( artifactRepository, @@ -1011,6 +1014,7 @@ interface LifecycleManagerDeps { wsManager: SessionWebSocketManager; alarmScheduler: RehydratableAlarmScheduler; sandboxDashboardSettings: SandboxDashboardSettings; + backgroundTasks: BackgroundTasks; } /** Create the lifecycle manager with all required adapters. */ @@ -1028,6 +1032,7 @@ function createLifecycleManager(deps: LifecycleManagerDeps): SandboxLifecycleMan wsManager, alarmScheduler, sandboxDashboardSettings, + backgroundTasks, } = deps; // Both throw on a misconfigured deployment — deliberately at graph // construction, so every session request fails at initialization instead of @@ -1131,6 +1136,7 @@ function createLifecycleManager(deps: LifecycleManagerDeps): SandboxLifecycleMan idGenerator, shutdown, config, - imageBuildLookup + imageBuildLookup, + backgroundTasks ); } diff --git a/packages/control-plane/src/session/sandbox-events/processor.test.ts b/packages/control-plane/src/session/sandbox-events/processor.test.ts index 8b13d50523..fe64a7fdac 100644 --- a/packages/control-plane/src/session/sandbox-events/processor.test.ts +++ b/packages/control-plane/src/session/sandbox-events/processor.test.ts @@ -120,6 +120,7 @@ function createProcessor( deliverTransition: vi.fn(async () => {}), }; const usageRepository = { recordStepUsage: vi.fn() }; + const refreshMetricsAfterStep = vi.fn((_messageId: string | null) => {}); // The real family composition, mirroring components.ts, so the suite keeps // pinning end-to-end processSandboxEvent behavior across the split. @@ -135,7 +136,8 @@ function createProcessor( messenger, updateLastActivity, budgetService as unknown as SessionBudgetService, - persistedUsage ?? (usageRepository as unknown as UsageRepository) + persistedUsage ?? (usageRepository as unknown as UsageRepository), + refreshMetricsAfterStep ), new SandboxArtifactEventHandler( artifactRepository, @@ -203,6 +205,7 @@ function createProcessor( log, budgetService, usageRepository, + refreshMetricsAfterStep, }; } @@ -515,6 +518,46 @@ describe("SessionSandboxEventProcessor", () => { } ); + it("keeps a recorded step's result when refreshing its metrics fails", async () => { + const h = createProcessor(); + const refreshError = new Error("Malformed persisted session row"); + h.refreshMetricsAfterStep.mockImplementation(() => { + throw refreshError; + }); + + await expect( + h.processor.processSandboxEvent({ + type: "step_finish", + messageId: "msg-1", + sandboxId: "sb-1", + timestamp: 1000, + tokens: { input: 10 }, + }) + ).resolves.toBeUndefined(); + expect(h.backgroundTasks.failures).toEqual([refreshError]); + }); + + it("reports the usage error rather than a failed metrics refresh", async () => { + const h = createProcessor(); + const persistenceError = new Error("usage write failed"); + h.usageRepository.recordStepUsage.mockImplementationOnce(() => { + throw persistenceError; + }); + h.refreshMetricsAfterStep.mockImplementation(() => { + throw new Error("Malformed persisted session row"); + }); + + await expect( + h.processor.processSandboxEvent({ + type: "step_finish", + messageId: "msg-1", + sandboxId: "sb-1", + timestamp: 1000, + cost: 0.25, + }) + ).rejects.toBe(persistenceError); + }); + it("records unavailable cost tracking for positive-token steps without cost", async () => { const h = createProcessor(); const event: SandboxEvent = { diff --git a/packages/control-plane/src/session/sandbox-events/streaming.handler.ts b/packages/control-plane/src/session/sandbox-events/streaming.handler.ts index d3631913e2..3c9ea4b0db 100644 --- a/packages/control-plane/src/session/sandbox-events/streaming.handler.ts +++ b/packages/control-plane/src/session/sandbox-events/streaming.handler.ts @@ -13,8 +13,10 @@ import { persistSandboxEvent, type SandboxEventContext } from "./context"; * execution (tokens, steps, tool activity, compaction). Every event here is * broadcast to clients; the ones with a durable representation also record * to the timeline (steps renew activity, accumulate cost, and persist usage). Nothing - * here transitions session state. Also owns the timeline-observer path - * (`recordTimelineEvent`) for events that persist and broadcast unchanged. + * here transitions session state; a step whose turn has already ended + * refreshes the metrics projection itself, as no settle for that turn is still + * to come. Also owns the timeline-observer path (`recordTimelineEvent`) for + * events that persist and broadcast unchanged. */ export class SandboxStreamingEventHandler { constructor( @@ -24,7 +26,8 @@ export class SandboxStreamingEventHandler { private readonly messenger: SessionMessenger, private readonly updateLastActivity: (timestamp: number) => void, private readonly budgetService: SessionBudgetService, - private readonly usageRepository: UsageRepository + private readonly usageRepository: UsageRepository, + private readonly refreshMetricsAfterStep: (messageId: string | null) => void ) {} handleToken(event: Extract, context: SandboxEventContext): void { @@ -56,19 +59,28 @@ export class SandboxStreamingEventHandler { this.updateLastActivity(context.now); this.messenger.broadcast({ type: "sandbox_event", event }); if (event.type === "step_finish") { - let persistenceFailure: { error: unknown } | null = null; try { - this.usageRepository.recordStepUsage(event, context.messageId, context.now); - } catch (error) { - persistenceFailure = { error }; - } - try { - await this.budgetService.ingestStepFinish(event, context.messageId, context.now); - } catch (error) { + let persistenceFailure: { error: unknown } | null = null; + try { + this.usageRepository.recordStepUsage(event, context.messageId, context.now); + } catch (error) { + persistenceFailure = { error }; + } + try { + await this.budgetService.ingestStepFinish(event, context.messageId, context.now); + } catch (error) { + if (persistenceFailure) throw persistenceFailure.error; + throw error; + } if (persistenceFailure) throw persistenceFailure.error; - throw error; + } finally { + // Submitted so a failed refresh is logged at the task boundary rather + // than replacing the step's own outcome. + this.backgroundTasks.submit(async () => this.refreshMetricsAfterStep(context.messageId), { + name: "session_index.refresh_step_metrics", + context: { message_id: context.messageId }, + }); } - if (persistenceFailure) throw persistenceFailure.error; } } diff --git a/packages/control-plane/src/session/sandbox-repository.test.ts b/packages/control-plane/src/session/sandbox-repository.test.ts index 08914294b4..cf8cb05704 100644 --- a/packages/control-plane/src/session/sandbox-repository.test.ts +++ b/packages/control-plane/src/session/sandbox-repository.test.ts @@ -575,6 +575,30 @@ describe("SandboxRepository boot state (SQLite)", () => { }); }); + it("resolves a VM only while its generation and pending handle still match", async () => { + const { repository, set } = createSqliteRepository(); + const generation = { sandboxId: "sb-1", createdAt: 1000 }; + const access = { + providerObjectId: "sb-real", + codeServer: { url: "https://editor.example", password: "secret" }, + vnc: null, + ttyd: null, + tunnelUrls: { "8080": "https://port.example" }, + }; + set("status = 'connecting', modal_sandbox_id = 'sb-1', modal_object_id = 'pending'"); + expect(await repository.completeProviderResume(generation, access, "other")).toBe(false); + expect(repository.getSandbox()?.modal_object_id).toBe("pending"); + expect(await repository.completeProviderResume(generation, access, "pending")).toBe(true); + expect(repository.getSandbox()).toMatchObject({ + modal_object_id: "sb-real", + code_server_url: "https://editor.example", + }); + expect(await repository.getSandboxAccessSecret("codeServer")).toBe("secret"); + set("modal_sandbox_id = 'sb-2', created_at = 2000, modal_object_id = 'pending'"); + expect(await repository.completeProviderResume(generation, access, "pending")).toBe(false); + expect(repository.getSandbox()?.modal_object_id).toBe("pending"); + }); + describe("markSandboxReady", () => { const generation = { sandboxId: "sb-1", createdAt: 1000 }; diff --git a/packages/control-plane/src/session/sandbox-repository.ts b/packages/control-plane/src/session/sandbox-repository.ts index 987a38422e..fcc392c7b9 100644 --- a/packages/control-plane/src/session/sandbox-repository.ts +++ b/packages/control-plane/src/session/sandbox-repository.ts @@ -382,7 +382,8 @@ export class SandboxRepository { */ async completeProviderResume( generation: { sandboxId: string | null; createdAt: number }, - access: ProviderResumeAccessData + access: ProviderResumeAccessData, + expectedProviderObjectId?: string ): Promise { const [codeServerPassword, vncPassword, ttydToken] = await Promise.all([ access.codeServer ? this.encrypt(access.codeServer.password) : null, @@ -401,7 +402,8 @@ export class SandboxRepository { tunnel_urls = ? WHERE id = (SELECT id FROM sandbox LIMIT 1) AND modal_sandbox_id IS ? AND created_at = ? - AND status IN ('connecting', 'ready') AND fenced = 0`, + AND (status IN ('connecting', 'ready') OR (? IS NOT NULL AND status = 'spawning')) + AND fenced = 0 AND (? IS NULL OR modal_object_id = ?)`, access.providerObjectId, access.codeServer?.url ?? null, codeServerPassword, @@ -411,7 +413,10 @@ export class SandboxRepository { ttydToken, access.tunnelUrls ? JSON.stringify(access.tunnelUrls) : null, generation.sandboxId, - generation.createdAt + generation.createdAt, + expectedProviderObjectId ?? null, + expectedProviderObjectId ?? null, + expectedProviderObjectId ?? null ); result.toArray(); return (result.rowsWritten ?? 0) > 0; diff --git a/packages/control-plane/src/session/sandbox-shutdown.ts b/packages/control-plane/src/session/sandbox-shutdown.ts index 8eeaba5754..6bae341944 100644 --- a/packages/control-plane/src/session/sandbox-shutdown.ts +++ b/packages/control-plane/src/session/sandbox-shutdown.ts @@ -295,6 +295,25 @@ export class SandboxShutdownCoordinator { this.notifyLifecycleChange(); } + /** Bridge recovery replaces only the pending handle, leaving the lifetime and restore hold intact. */ + recordResolvedProviderHandle( + generation: SandboxGeneration, + expectedReference: string, + providerObjectId: string + ): void { + const state = this.deps.store.read(); + const row = this.deps.sandbox.getSandbox(); + if ( + !state || + !this.current(state) || + !this.matches(state, generation) || + state.providerObjectId !== expectedReference || + row?.modal_object_id !== providerObjectId + ) + return; + this.publish({ ...state, providerObjectId }); + } + runtimeReady(version?: 1): void { const state = this.deps.store.read(); if (!state || !this.current(state)) return; @@ -410,6 +429,23 @@ export class SandboxShutdownCoordinator { : { kind: "normal" }; } + /** + * Holds a failed boot of the source a retained receipt names, as a failed + * retained resume is held, and records that source so recovery can retire + * and resume it: an ordinary resume reserves its generation without a + * provider handle. Holds nothing and resolves false for any other object. + */ + holdFailedRetainedBoot(error: string, generation: SandboxGeneration): boolean { + const state = this.deps.store.read(); + const source = state?.receipt?.kind === "retained" ? state.receipt.artifactId : null; + if (!state || !source || this.deps.sandbox.getSandbox()?.modal_object_id !== source) + return false; + if (this.current(state) && this.matches(state, generation)) + this.deps.store.write({ ...state, providerObjectId: source }); + this.holdFailedRecovery(error, generation); + return true; + } + holdFailedRecovery(error: string, generation?: SandboxGeneration): void { const state = this.deps.store.read(); const row = this.deps.sandbox.getSandbox(); diff --git a/packages/control-plane/src/session/session-status-service.test.ts b/packages/control-plane/src/session/session-status-service.test.ts index dfb177885b..f80ab09ee4 100644 --- a/packages/control-plane/src/session/session-status-service.test.ts +++ b/packages/control-plane/src/session/session-status-service.test.ts @@ -63,6 +63,7 @@ function harness(options: { session?: SessionRow | null } = {}) { updateSessionStatus: vi.fn(), getPendingOrProcessingCount: vi.fn(() => 0), getLatestTerminalMessage: vi.fn(() => null as MessageRow | null), + getMessageStatus: vi.fn((_messageId: string): MessageRow["status"] | null => "failed"), getMessageCount: vi.fn(() => 3), getActiveDurationMs: vi.fn(() => 4500), }; @@ -292,6 +293,82 @@ describe("SessionStatusService.transition", () => { expect(h.sessionIndex.updateMetrics).not.toHaveBeenCalled(); }); + it("defers a step of the processing turn but projects one whose turn has ended", () => { + // A budget stop leaves a queued prompt that keeps the session active. + const h = harness({ session: createSession({ status: "active" }) }); + h.repository.getMessageStatus.mockReturnValueOnce("processing"); + + h.service.refreshMetricsAfterStep("msg-running"); + expect(h.sessionIndex.updateMetrics).not.toHaveBeenCalled(); + + h.service.refreshMetricsAfterStep("msg-stopped"); + expect(h.sessionIndex.updateMetrics).toHaveBeenCalledWith( + "public-session-1", + expect.objectContaining({ inputTokens: 1200 }) + ); + }); + + it("writes usage that lands during a metrics write after it, never beside it", async () => { + const h = harness({ session: createSession({ status: "failed" }) }); + let releaseFirstWrite!: () => void; + h.sessionIndex.updateMetrics.mockImplementationOnce( + () => new Promise((resolve) => (releaseFirstWrite = () => resolve(true))) + ); + + expect(await h.service.transition("failed")).toBe(false); + h.usageRepository.getSessionTotals.mockReturnValue(createUsageTotals({ inputTokens: 1500 })); + h.service.refreshMetricsAfterStep("msg-1"); + h.service.refreshMetricsAfterStep("msg-1"); + + expect(h.sessionIndex.updateMetrics).toHaveBeenCalledTimes(1); + releaseFirstWrite(); + await h.backgroundTasks.settle(); + + expect(h.sessionIndex.updateMetrics).toHaveBeenCalledTimes(2); + expect(h.sessionIndex.updateMetrics).toHaveBeenLastCalledWith( + "public-session-1", + expect.objectContaining({ inputTokens: 1500 }) + ); + }); + + it("still writes usage that landed during a metrics write that failed", async () => { + const h = harness({ session: createSession({ status: "failed" }) }); + const error = new Error("d1 down"); + let failFirstWrite!: () => void; + h.sessionIndex.updateMetrics.mockImplementationOnce( + () => new Promise((_resolve, reject) => (failFirstWrite = () => reject(error))) + ); + + expect(await h.service.transition("failed")).toBe(false); + h.usageRepository.getSessionTotals.mockReturnValue(createUsageTotals({ inputTokens: 1500 })); + h.service.refreshMetricsAfterStep("msg-1"); + failFirstWrite(); + await h.backgroundTasks.settle(); + + expect(h.sessionIndex.updateMetrics).toHaveBeenCalledTimes(2); + expect(h.sessionIndex.updateMetrics).toHaveBeenLastCalledWith( + "public-session-1", + expect.objectContaining({ inputTokens: 1500 }) + ); + expect(h.backgroundTasks.failures).toEqual([error]); + }); + + it("does not retry a failed metrics write when nothing newer is pending", async () => { + const h = harness({ session: createSession({ status: "failed" }) }); + const error = new Error("d1 down"); + h.sessionIndex.updateMetrics.mockRejectedValueOnce(error); + + expect(await h.service.transition("failed")).toBe(false); + await h.backgroundTasks.settle(); + + expect(h.sessionIndex.updateMetrics).toHaveBeenCalledTimes(1); + expect(h.backgroundTasks.failures).toEqual([error]); + + h.service.refreshMetricsAfterStep("msg-1"); + await h.backgroundTasks.settle(); + expect(h.sessionIndex.updateMetrics).toHaveBeenCalledTimes(2); + }); + it("logs index sync failures without throwing", async () => { const h = harness({ session: createSession({ status: "created" }) }); h.statusProjection.project.mockRejectedValue(new Error("d1 down")); diff --git a/packages/control-plane/src/session/session-status-service.ts b/packages/control-plane/src/session/session-status-service.ts index 191ca7546f..7544d7f4f9 100644 --- a/packages/control-plane/src/session/session-status-service.ts +++ b/packages/control-plane/src/session/session-status-service.ts @@ -27,6 +27,11 @@ import { isSessionPromptable, isTurnSettled } from "@open-inspect/shared/types/s type SessionIndexProjections = Pick; export class SessionStatusService { + /** A metrics write is in flight; later requests fold into its next pass. */ + private metricsSyncInFlight = false; + /** State changed after the in-flight write read it. */ + private metricsSyncStale = false; + constructor( private readonly backgroundTasks: BackgroundTasks, private readonly log: Logger, @@ -170,6 +175,24 @@ export class SessionStatusService { return true; } + /** + * Re-project metrics for a step whose turn is no longer processing. A stop + * ends the turn before the sandbox has seen the stop, so a step already in + * flight lands afterwards, and the sandbox's own terminal for that turn then + * settles nothing. The turn decides, not the session: a budget stop leaves a + * queued prompt that keeps the session `active` but cannot dispatch, so no + * later settle would cover the step. A step of the processing turn waits + * for the next settle. + */ + refreshMetricsAfterStep(messageId: string | null): void { + if (messageId !== null && this.messageRepository.getMessageStatus(messageId) === "processing") { + return; + } + const session = this.repository.getSession(); + if (!session) return; + this.syncSessionMetrics(this.getPublicSessionId(session)); + } + private async projectTransition( session: SessionRow, publicSessionId: string, @@ -338,30 +361,34 @@ export class SessionStatusService { }); } + /** + * Writes are last-write-wins, so at most one is in flight, and each reads + * the session when it runs: a request made during a write only marks it + * stale, and the write goes round again with the newer state instead of + * racing it to D1. A failed pass still goes round when a newer request + * arrived during it; the first failure is reported once the writer drains. + */ private syncSessionMetrics(sessionId: string): void { - const session = this.repository.getSession(); - if (!session) return; - - const messageCount = this.messageRepository.getMessageCount(); - const activeDurationMs = this.messageRepository.getActiveDurationMs(); - const artifacts = this.artifactRepository.listArtifacts(); - const prCount = artifacts.filter((a) => a.type === "pr").length; + if (this.metricsSyncInFlight) { + this.metricsSyncStale = true; + return; + } + if (!this.repository.getSession()) return; + this.metricsSyncInFlight = true; this.backgroundTasks.submit( - () => { - // The index keeps aggregate-friendly zeros; "unknown" lives in the usage rows. - const tokens = this.usageRepository.getSessionTotals(); - return this.sessionIndex.updateMetrics(sessionId, { - totalCost: session.total_cost ?? 0, - activeDurationMs, - messageCount, - prCount, - inputTokens: tokens.inputTokens ?? 0, - outputTokens: tokens.outputTokens ?? 0, - reasoningTokens: tokens.reasoningTokens ?? 0, - cacheReadTokens: tokens.cacheReadTokens ?? 0, - cacheWriteTokens: tokens.cacheWriteTokens ?? 0, - }); + async () => { + let failure: { error: unknown } | null = null; + do { + this.metricsSyncStale = false; + try { + await this.projectSessionMetrics(sessionId); + } catch (error) { + failure ??= { error }; + } + } while (this.metricsSyncStale); + this.metricsSyncInFlight = false; + if (failure) throw failure.error; }, { name: "session_index.update_metrics", @@ -369,4 +396,27 @@ export class SessionStatusService { } ); } + + private async projectSessionMetrics(sessionId: string): Promise { + const session = this.repository.getSession(); + if (!session) return; + + const messageCount = this.messageRepository.getMessageCount(); + const activeDurationMs = this.messageRepository.getActiveDurationMs(); + const artifacts = this.artifactRepository.listArtifacts(); + const prCount = artifacts.filter((a) => a.type === "pr").length; + // The index keeps aggregate-friendly zeros; "unknown" lives in the usage rows. + const tokens = this.usageRepository.getSessionTotals(); + await this.sessionIndex.updateMetrics(sessionId, { + totalCost: session.total_cost ?? 0, + activeDurationMs, + messageCount, + prCount, + inputTokens: tokens.inputTokens ?? 0, + outputTokens: tokens.outputTokens ?? 0, + reasoningTokens: tokens.reasoningTokens ?? 0, + cacheReadTokens: tokens.cacheReadTokens ?? 0, + cacheWriteTokens: tokens.cacheWriteTokens ?? 0, + }); + } } diff --git a/packages/control-plane/src/types.ts b/packages/control-plane/src/types.ts index e9c1729ea3..ace14c9763 100644 --- a/packages/control-plane/src/types.ts +++ b/packages/control-plane/src/types.ts @@ -105,6 +105,7 @@ export interface EnvConfig { SANDBOX_BOOT_TIMEOUT_MS?: string; // Longest a connected sandbox may boot before it is failed, in ms; defaults to DEFAULT_BOOT_BUDGET_CONFIG EXECUTION_TIMEOUT_MS?: string; // Max processing time for one message before auto-fail, for sessions and for the automation runs watching them; overridden per session by sandboxTimeoutMs, and falls back to DEFAULT_SANDBOX_TIMEOUT_SECONDS SECRETS_CAP_ENFORCEMENT?: string; // "enforce" (default) fails spawn/build on oversized secret payloads; set "warn" to only log + TEAMS_ENFORCEMENT?: string; // "off" | "shadow" (default) | "on" // Logging LOG_LEVEL?: string; // "debug" | "info" | "warn" | "error" (default: "info") diff --git a/packages/control-plane/test/integration/__snapshots__/hono-route-catalog-conformance.test.ts.snap b/packages/control-plane/test/integration/__snapshots__/hono-route-catalog-conformance.test.ts.snap index 5255cfb34e..2ea9203883 100644 --- a/packages/control-plane/test/integration/__snapshots__/hono-route-catalog-conformance.test.ts.snap +++ b/packages/control-plane/test/integration/__snapshots__/hono-route-catalog-conformance.test.ts.snap @@ -24,49 +24,49 @@ exports[`Hono route catalog conformance > dispatches every frozen method/path/po "{"identity":"POST /sessions","pathname":"/sessions","groups":{},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.create"}],"auditAllowed":true,"service":{"kind":"actor"}},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":true}", "{"identity":"GET /sessions","pathname":"/sessions","groups":{},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.read"}],"auditAllowed":false,"service":{"kind":"actor"}},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", "{"identity":"GET /sessions/inbox","pathname":"/sessions/inbox","groups":{},"authentication":"user","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.read"}],"auditAllowed":false,"service":{"kind":"deny"}},"supportedScmProviders":"all","cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"PATCH /sessions/:id/read-state","pathname":"/sessions/fixture-22-id%2Fraw/read-state","groups":{"id":"fixture-22-id%2Fraw"},"authentication":"user","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.read"}],"auditAllowed":false,"service":{"kind":"actor"}},"supportedScmProviders":"all","cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"DELETE /sessions/:id","pathname":"/sessions/fixture-23-id%2Fraw","groups":{"id":"fixture-23-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.delete"}],"auditAllowed":true,"service":{"kind":"actor"}},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"PATCH /sessions/:id/read-state","pathname":"/sessions/fixture-22-id%2Fraw/read-state","groups":{"id":"fixture-22-id%2Fraw"},"authentication":"user","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"read"}],"service":{"kind":"actor"},"auditAllowed":false},"supportedScmProviders":"all","cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"DELETE /sessions/:id","pathname":"/sessions/fixture-23-id%2Fraw","groups":{"id":"fixture-23-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"delete"}],"service":{"kind":"actor"},"auditAllowed":true},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", "{"identity":"GET /sessions/export","pathname":"/sessions/export","groups":{},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.export"}],"auditAllowed":true,"service":{"kind":"actor"}},"supportedScmProviders":"all","cacheControl":"private, no-store","hasServiceActorClaims":false}", - "{"identity":"GET /sessions/:id/export","pathname":"/sessions/fixture-25-id%2Fraw/export","groups":{"id":"fixture-25-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.export"}],"auditAllowed":true,"service":{"kind":"actor"}},"supportedScmProviders":"all","cacheControl":"private, no-store","hasServiceActorClaims":false}", - "{"identity":"GET /sessions/:id/sandbox-access","pathname":"/sessions/fixture-26-id%2Fraw/sandbox-access","groups":{"id":"fixture-26-id%2Fraw"},"authentication":"user","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.sandbox_access"}],"auditAllowed":true,"service":{"kind":"actor"}},"supportedScmProviders":"all","cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"GET /sessions/:id","pathname":"/sessions/fixture-27-id%2Fraw","groups":{"id":"fixture-27-id%2Fraw"},"authentication":"user","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.read"}],"auditAllowed":false,"service":{"kind":"actor"}},"supportedScmProviders":"all","cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"POST /sessions/:id/stop","pathname":"/sessions/fixture-28-id%2Fraw/stop","groups":{"id":"fixture-28-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.lifecycle"}],"auditAllowed":true,"service":{"kind":"actor","actorlessGrants":[{"service":"linear-bot"}]}},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"GET /sessions/:id/export","pathname":"/sessions/fixture-25-id%2Fraw/export","groups":{"id":"fixture-25-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"read"},{"kind":"permission","permission":"sessions.export"}],"service":{"kind":"actor"},"auditAllowed":true},"supportedScmProviders":"all","cacheControl":"private, no-store","hasServiceActorClaims":false}", + "{"identity":"GET /sessions/:id/sandbox-access","pathname":"/sessions/fixture-26-id%2Fraw/sandbox-access","groups":{"id":"fixture-26-id%2Fraw"},"authentication":"user","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"sandbox"}],"service":{"kind":"actor"},"auditAllowed":true},"supportedScmProviders":"all","cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"GET /sessions/:id","pathname":"/sessions/fixture-27-id%2Fraw","groups":{"id":"fixture-27-id%2Fraw"},"authentication":"user","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"read"}],"service":{"kind":"actor"},"auditAllowed":false},"supportedScmProviders":"all","cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"POST /sessions/:id/stop","pathname":"/sessions/fixture-28-id%2Fraw/stop","groups":{"id":"fixture-28-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"lifecycle"}],"service":{"kind":"actor","actorlessGrants":[{"service":"linear-bot"}]},"auditAllowed":true},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", "{"identity":"POST /sessions/:id/sandbox-error","pathname":"/sessions/fixture-29-id%2Fraw/sandbox-error","groups":{"id":"fixture-29-id%2Fraw"},"authentication":"handler-authenticated","authorization":{"kind":"none","auditAllowed":false},"supportedScmProviders":"all","cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"GET /sessions/:id/events","pathname":"/sessions/fixture-30-id%2Fraw/events","groups":{"id":"fixture-30-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.read"}],"auditAllowed":false,"service":{"kind":"actor","actorlessGrants":[{"service":"slack-bot"},{"service":"linear-bot"}]}},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"GET /sessions/:id/artifacts","pathname":"/sessions/fixture-31-id%2Fraw/artifacts","groups":{"id":"fixture-31-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.read"}],"auditAllowed":false,"service":{"kind":"actor","actorlessGrants":[{"service":"slack-bot"},{"service":"linear-bot"}]}},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"GET /sessions/:id/participants","pathname":"/sessions/fixture-32-id%2Fraw/participants","groups":{"id":"fixture-32-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.read"}],"auditAllowed":false,"service":{"kind":"actor"}},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"GET /sessions/:id/participant-profiles","pathname":"/sessions/fixture-33-id%2Fraw/participant-profiles","groups":{"id":"fixture-33-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.read"}],"auditAllowed":false,"service":{"kind":"actor"}},"supportedScmProviders":"all","cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"GET /sessions/:id/messages","pathname":"/sessions/fixture-34-id%2Fraw/messages","groups":{"id":"fixture-34-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.read"}],"auditAllowed":false,"service":{"kind":"actor"}},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"POST /sessions/:id/pr","pathname":"/sessions/fixture-35-id%2Fraw/pr","groups":{"id":"fixture-35-id%2Fraw"},"authentication":"user-or-service-with-sandbox-fallback","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.collaborate"}],"auditAllowed":true,"service":{"kind":"actor"}},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"GET /sessions/:id/events","pathname":"/sessions/fixture-30-id%2Fraw/events","groups":{"id":"fixture-30-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"read"}],"service":{"kind":"actor","actorlessGrants":[{"service":"slack-bot"},{"service":"linear-bot"}]},"auditAllowed":false},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"GET /sessions/:id/artifacts","pathname":"/sessions/fixture-31-id%2Fraw/artifacts","groups":{"id":"fixture-31-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"read"}],"service":{"kind":"actor","actorlessGrants":[{"service":"slack-bot"},{"service":"linear-bot"}]},"auditAllowed":false},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"GET /sessions/:id/participants","pathname":"/sessions/fixture-32-id%2Fraw/participants","groups":{"id":"fixture-32-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"read"}],"service":{"kind":"actor"},"auditAllowed":false},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"GET /sessions/:id/participant-profiles","pathname":"/sessions/fixture-33-id%2Fraw/participant-profiles","groups":{"id":"fixture-33-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"read"}],"service":{"kind":"actor"},"auditAllowed":false},"supportedScmProviders":"all","cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"GET /sessions/:id/messages","pathname":"/sessions/fixture-34-id%2Fraw/messages","groups":{"id":"fixture-34-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"read"}],"service":{"kind":"actor"},"auditAllowed":false},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"POST /sessions/:id/pr","pathname":"/sessions/fixture-35-id%2Fraw/pr","groups":{"id":"fixture-35-id%2Fraw"},"authentication":"user-or-service-with-sandbox-fallback","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"collaborate"}],"service":{"kind":"actor"},"auditAllowed":true},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", "{"identity":"POST /sessions/:id/openai-token-refresh","pathname":"/sessions/fixture-36-id%2Fraw/openai-token-refresh","groups":{"id":"fixture-36-id%2Fraw"},"authentication":"sandbox","authorization":{"kind":"none","auditAllowed":false},"supportedScmProviders":"all","cacheControl":null,"hasServiceActorClaims":false}", "{"identity":"POST /sessions/:id/xai-token-refresh","pathname":"/sessions/fixture-37-id%2Fraw/xai-token-refresh","groups":{"id":"fixture-37-id%2Fraw"},"authentication":"sandbox","authorization":{"kind":"none","auditAllowed":false},"supportedScmProviders":"all","cacheControl":null,"hasServiceActorClaims":false}", "{"identity":"POST /sessions/:id/scm-credentials","pathname":"/sessions/fixture-38-id%2Fraw/scm-credentials","groups":{"id":"fixture-38-id%2Fraw"},"authentication":"sandbox","authorization":{"kind":"none","auditAllowed":false},"supportedScmProviders":["github","gitlab"],"cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"GET /sessions/:id/tunnel-urls","pathname":"/sessions/fixture-39-id%2Fraw/tunnel-urls","groups":{"id":"fixture-39-id%2Fraw"},"authentication":"user-or-service-with-sandbox-fallback","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.sandbox_access"}],"auditAllowed":true,"service":{"kind":"actor"}},"supportedScmProviders":"all","cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"PATCH /sessions/:id/title","pathname":"/sessions/fixture-40-id%2Fraw/title","groups":{"id":"fixture-40-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.lifecycle"}],"auditAllowed":true,"service":{"kind":"actor"}},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"POST /sessions/:id/archive","pathname":"/sessions/fixture-41-id%2Fraw/archive","groups":{"id":"fixture-41-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.lifecycle"}],"auditAllowed":true,"service":{"kind":"actor"}},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"POST /sessions/:id/unarchive","pathname":"/sessions/fixture-42-id%2Fraw/unarchive","groups":{"id":"fixture-42-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.lifecycle"}],"auditAllowed":true,"service":{"kind":"actor"}},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"PATCH /sessions/:id/budget","pathname":"/sessions/fixture-43-id%2Fraw/budget","groups":{"id":"fixture-43-id%2Fraw"},"authentication":"user","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.lifecycle"}],"auditAllowed":true,"service":{"kind":"actor"}},"supportedScmProviders":"all","cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"GET /sessions/:id/tunnel-urls","pathname":"/sessions/fixture-39-id%2Fraw/tunnel-urls","groups":{"id":"fixture-39-id%2Fraw"},"authentication":"user-or-service-with-sandbox-fallback","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"sandbox"}],"service":{"kind":"actor"},"auditAllowed":true},"supportedScmProviders":"all","cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"PATCH /sessions/:id/title","pathname":"/sessions/fixture-40-id%2Fraw/title","groups":{"id":"fixture-40-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"lifecycle"}],"service":{"kind":"actor"},"auditAllowed":true},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"POST /sessions/:id/archive","pathname":"/sessions/fixture-41-id%2Fraw/archive","groups":{"id":"fixture-41-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"lifecycle"}],"service":{"kind":"actor"},"auditAllowed":true},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"POST /sessions/:id/unarchive","pathname":"/sessions/fixture-42-id%2Fraw/unarchive","groups":{"id":"fixture-42-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"lifecycle"}],"service":{"kind":"actor"},"auditAllowed":true},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"PATCH /sessions/:id/budget","pathname":"/sessions/fixture-43-id%2Fraw/budget","groups":{"id":"fixture-43-id%2Fraw"},"authentication":"user","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"lifecycle"}],"service":{"kind":"actor"},"auditAllowed":true},"supportedScmProviders":"all","cacheControl":null,"hasServiceActorClaims":false}", "{"identity":"POST /sessions/batch-archive","pathname":"/sessions/batch-archive","groups":{},"authentication":"user","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.bulk_archive"}],"auditAllowed":true,"service":{"kind":"deny"}},"supportedScmProviders":"all","cacheControl":"private, no-store","hasServiceActorClaims":false}", - "{"identity":"POST /sessions/:id/ws-token","pathname":"/sessions/fixture-45-id%2Fraw/ws-token","groups":{"id":"fixture-45-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.read"}],"auditAllowed":false,"service":{"kind":"actor"}},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"POST /sessions/:id/prompt","pathname":"/sessions/fixture-46-id%2Fraw/prompt","groups":{"id":"fixture-46-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.collaborate"}],"auditAllowed":true,"service":{"kind":"actor"}},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"POST /sessions/:id/pull-requests/refresh","pathname":"/sessions/fixture-47-id%2Fraw/pull-requests/refresh","groups":{"id":"fixture-47-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.lifecycle"}],"auditAllowed":true,"service":{"kind":"actor"}},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"POST /sessions/:id/media","pathname":"/sessions/fixture-48-id%2Fraw/media","groups":{"id":"fixture-48-id%2Fraw"},"authentication":"user-or-service-with-sandbox-fallback","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.collaborate"}],"auditAllowed":true,"service":{"kind":"actor"}},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"GET /sessions/:id/media/:artifactId","pathname":"/sessions/fixture-49-id%2Fraw/media/fixture-49-artifactId%2Fraw","groups":{"id":"fixture-49-id%2Fraw","artifactId":"fixture-49-artifactId%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.read"}],"auditAllowed":false,"service":{"kind":"actor","actorlessGrants":[{"service":"slack-bot"}]}},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"POST /sessions/:id/attachments","pathname":"/sessions/fixture-50-id%2Fraw/attachments","groups":{"id":"fixture-50-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.collaborate"}],"auditAllowed":true,"service":{"kind":"actor"}},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"GET /sessions/:id/attachments/:attachmentId","pathname":"/sessions/fixture-51-id%2Fraw/attachments/fixture-51-attachmentId%2Fraw","groups":{"id":"fixture-51-id%2Fraw","attachmentId":"fixture-51-attachmentId%2Fraw"},"authentication":"user-or-service-with-sandbox-fallback","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.read"}],"auditAllowed":false,"service":{"kind":"actor"}},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"GET /sessions/:id/diff","pathname":"/sessions/fixture-52-id%2Fraw/diff","groups":{"id":"fixture-52-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.read"}],"auditAllowed":false,"service":{"kind":"actor"}},"supportedScmProviders":"all","cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"PUT /sessions/:id/diff","pathname":"/sessions/fixture-53-id%2Fraw/diff","groups":{"id":"fixture-53-id%2Fraw"},"authentication":"user-or-service-with-sandbox-fallback","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.collaborate"}],"auditAllowed":true,"service":{"kind":"actor"}},"supportedScmProviders":"all","cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"POST /sessions/:id/diff/failure","pathname":"/sessions/fixture-54-id%2Fraw/diff/failure","groups":{"id":"fixture-54-id%2Fraw"},"authentication":"user-or-service-with-sandbox-fallback","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.collaborate"}],"auditAllowed":true,"service":{"kind":"actor"}},"supportedScmProviders":"all","cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"GET /sessions/:id/diff/:revisionId/files/:fileId","pathname":"/sessions/fixture-55-id%2Fraw/diff/fixture-55-revisionId%2Fraw/files/fixture-55-fileId%2Fraw","groups":{"id":"fixture-55-id%2Fraw","revisionId":"fixture-55-revisionId%2Fraw","fileId":"fixture-55-fileId%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.read"}],"auditAllowed":false,"service":{"kind":"actor"}},"supportedScmProviders":"all","cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"POST /sessions/:id/diff/retry","pathname":"/sessions/fixture-56-id%2Fraw/diff/retry","groups":{"id":"fixture-56-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.lifecycle"}],"auditAllowed":true,"service":{"kind":"actor"}},"supportedScmProviders":"all","cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"GET /sessions/:id/skills","pathname":"/sessions/fixture-57-id%2Fraw/skills","groups":{"id":"fixture-57-id%2Fraw"},"authentication":"user","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.read"}],"auditAllowed":false,"service":{"kind":"actor"}},"supportedScmProviders":"all","cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"POST /sessions/:id/ws-token","pathname":"/sessions/fixture-45-id%2Fraw/ws-token","groups":{"id":"fixture-45-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"read"}],"service":{"kind":"actor"},"auditAllowed":false},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"POST /sessions/:id/prompt","pathname":"/sessions/fixture-46-id%2Fraw/prompt","groups":{"id":"fixture-46-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"collaborate"}],"service":{"kind":"actor"},"auditAllowed":true},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"POST /sessions/:id/pull-requests/refresh","pathname":"/sessions/fixture-47-id%2Fraw/pull-requests/refresh","groups":{"id":"fixture-47-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"lifecycle"}],"service":{"kind":"actor"},"auditAllowed":true},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"POST /sessions/:id/media","pathname":"/sessions/fixture-48-id%2Fraw/media","groups":{"id":"fixture-48-id%2Fraw"},"authentication":"user-or-service-with-sandbox-fallback","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"collaborate"}],"service":{"kind":"actor"},"auditAllowed":true},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"GET /sessions/:id/media/:artifactId","pathname":"/sessions/fixture-49-id%2Fraw/media/fixture-49-artifactId%2Fraw","groups":{"id":"fixture-49-id%2Fraw","artifactId":"fixture-49-artifactId%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"read"}],"service":{"kind":"actor","actorlessGrants":[{"service":"slack-bot"}]},"auditAllowed":false},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"POST /sessions/:id/attachments","pathname":"/sessions/fixture-50-id%2Fraw/attachments","groups":{"id":"fixture-50-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"collaborate"}],"service":{"kind":"actor"},"auditAllowed":true},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"GET /sessions/:id/attachments/:attachmentId","pathname":"/sessions/fixture-51-id%2Fraw/attachments/fixture-51-attachmentId%2Fraw","groups":{"id":"fixture-51-id%2Fraw","attachmentId":"fixture-51-attachmentId%2Fraw"},"authentication":"user-or-service-with-sandbox-fallback","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"read"}],"service":{"kind":"actor"},"auditAllowed":false},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"GET /sessions/:id/diff","pathname":"/sessions/fixture-52-id%2Fraw/diff","groups":{"id":"fixture-52-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"read"}],"service":{"kind":"actor"},"auditAllowed":false},"supportedScmProviders":"all","cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"PUT /sessions/:id/diff","pathname":"/sessions/fixture-53-id%2Fraw/diff","groups":{"id":"fixture-53-id%2Fraw"},"authentication":"user-or-service-with-sandbox-fallback","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"collaborate"}],"service":{"kind":"actor"},"auditAllowed":true},"supportedScmProviders":"all","cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"POST /sessions/:id/diff/failure","pathname":"/sessions/fixture-54-id%2Fraw/diff/failure","groups":{"id":"fixture-54-id%2Fraw"},"authentication":"user-or-service-with-sandbox-fallback","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"collaborate"}],"service":{"kind":"actor"},"auditAllowed":true},"supportedScmProviders":"all","cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"GET /sessions/:id/diff/:revisionId/files/:fileId","pathname":"/sessions/fixture-55-id%2Fraw/diff/fixture-55-revisionId%2Fraw/files/fixture-55-fileId%2Fraw","groups":{"id":"fixture-55-id%2Fraw","revisionId":"fixture-55-revisionId%2Fraw","fileId":"fixture-55-fileId%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"read"}],"service":{"kind":"actor"},"auditAllowed":false},"supportedScmProviders":"all","cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"POST /sessions/:id/diff/retry","pathname":"/sessions/fixture-56-id%2Fraw/diff/retry","groups":{"id":"fixture-56-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"lifecycle"}],"service":{"kind":"actor"},"auditAllowed":true},"supportedScmProviders":"all","cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"GET /sessions/:id/skills","pathname":"/sessions/fixture-57-id%2Fraw/skills","groups":{"id":"fixture-57-id%2Fraw"},"authentication":"user","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"read"}],"service":{"kind":"actor"},"auditAllowed":false},"supportedScmProviders":"all","cacheControl":null,"hasServiceActorClaims":false}", "{"identity":"GET /sessions/:id/sandbox-skills","pathname":"/sessions/fixture-58-id%2Fraw/sandbox-skills","groups":{"id":"fixture-58-id%2Fraw"},"authentication":"sandbox","authorization":{"kind":"none","auditAllowed":false},"supportedScmProviders":"all","cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"POST /sessions/:id/children","pathname":"/sessions/fixture-59-id%2Fraw/children","groups":{"id":"fixture-59-id%2Fraw"},"authentication":"user-or-service-with-sandbox-fallback","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.create"},{"kind":"permission","permission":"sessions.collaborate"}],"service":{"kind":"actor"},"auditAllowed":true},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"GET /sessions/:id/children","pathname":"/sessions/fixture-60-id%2Fraw/children","groups":{"id":"fixture-60-id%2Fraw"},"authentication":"user-or-service-with-sandbox-fallback","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.read"}],"auditAllowed":false,"service":{"kind":"actor"}},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"GET /sessions/:id/children/:childId","pathname":"/sessions/fixture-61-id%2Fraw/children/fixture-61-childId%2Fraw","groups":{"id":"fixture-61-id%2Fraw","childId":"fixture-61-childId%2Fraw"},"authentication":"user-or-service-with-sandbox-fallback","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.read"}],"auditAllowed":false,"service":{"kind":"actor"}},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"POST /sessions/:id/children/:childId/cancel","pathname":"/sessions/fixture-62-id%2Fraw/children/fixture-62-childId%2Fraw/cancel","groups":{"id":"fixture-62-id%2Fraw","childId":"fixture-62-childId%2Fraw"},"authentication":"user-or-service-with-sandbox-fallback","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.lifecycle"}],"auditAllowed":true,"service":{"kind":"actor"}},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"POST /sessions/:id/children","pathname":"/sessions/fixture-59-id%2Fraw/children","groups":{"id":"fixture-59-id%2Fraw"},"authentication":"user-or-service-with-sandbox-fallback","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"collaborate"},{"kind":"permission","permission":"sessions.create"}],"service":{"kind":"actor"},"auditAllowed":true},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"GET /sessions/:id/children","pathname":"/sessions/fixture-60-id%2Fraw/children","groups":{"id":"fixture-60-id%2Fraw"},"authentication":"user-or-service-with-sandbox-fallback","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"read"}],"service":{"kind":"actor"},"auditAllowed":false},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"GET /sessions/:id/children/:childId","pathname":"/sessions/fixture-61-id%2Fraw/children/fixture-61-childId%2Fraw","groups":{"id":"fixture-61-id%2Fraw","childId":"fixture-61-childId%2Fraw"},"authentication":"user-or-service-with-sandbox-fallback","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"read"},{"kind":"session","sessionIdParam":"childId","action":"read"}],"service":{"kind":"actor"},"auditAllowed":false},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"POST /sessions/:id/children/:childId/cancel","pathname":"/sessions/fixture-62-id%2Fraw/children/fixture-62-childId%2Fraw/cancel","groups":{"id":"fixture-62-id%2Fraw","childId":"fixture-62-childId%2Fraw"},"authentication":"user-or-service-with-sandbox-fallback","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"read"},{"kind":"session","sessionIdParam":"childId","action":"lifecycle"}],"service":{"kind":"actor"},"auditAllowed":true},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", "{"identity":"POST /sessions/:id/children/:childId/prompt","pathname":"/sessions/fixture-63-id%2Fraw/children/fixture-63-childId%2Fraw/prompt","groups":{"id":"fixture-63-id%2Fraw","childId":"fixture-63-childId%2Fraw"},"authentication":"sandbox","authorization":{"kind":"none","auditAllowed":false},"supportedScmProviders":"all","cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"POST /sessions/:id/slack-notify","pathname":"/sessions/fixture-64-id%2Fraw/slack-notify","groups":{"id":"fixture-64-id%2Fraw"},"authentication":"user-or-service-with-sandbox-fallback","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.collaborate"}],"auditAllowed":true,"service":{"kind":"actor"}},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"POST /sessions/:id/slack-notify","pathname":"/sessions/fixture-64-id%2Fraw/slack-notify","groups":{"id":"fixture-64-id%2Fraw"},"authentication":"user-or-service-with-sandbox-fallback","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"collaborate"}],"service":{"kind":"actor"},"auditAllowed":true},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", "{"identity":"GET /repos","pathname":"/repos","groups":{},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"repositories.read"}],"auditAllowed":false,"service":{"kind":"actor","actorlessGrants":[{"service":"slack-bot"},{"service":"linear-bot"}]}},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", "{"identity":"PUT /repos/:owner/:name/metadata","pathname":"/repos/fixture-66-owner%2Fraw/fixture-66-name%2Fraw/metadata","groups":{"owner":"fixture-66-owner%2Fraw","name":"fixture-66-name%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"repositories.settings.manage"}],"auditAllowed":true,"service":{"kind":"actor"}},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", "{"identity":"GET /repos/:owner/:name/metadata","pathname":"/repos/fixture-67-owner%2Fraw/fixture-67-name%2Fraw/metadata","groups":{"owner":"fixture-67-owner%2Fraw","name":"fixture-67-name%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"repositories.read"}],"auditAllowed":false,"service":{"kind":"actor","actorlessGrants":[{"service":"github-bot"}]}},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", diff --git a/packages/control-plane/test/integration/__snapshots__/route-admission-matrix.test.ts.snap b/packages/control-plane/test/integration/__snapshots__/route-admission-matrix.test.ts.snap index a4f2e9ae76..919f68b447 100644 --- a/packages/control-plane/test/integration/__snapshots__/route-admission-matrix.test.ts.snap +++ b/packages/control-plane/test/integration/__snapshots__/route-admission-matrix.test.ts.snap @@ -436,3 +436,101 @@ exports[`route admission matrix > rejects every credentialed route anonymously b "DELETE /sessions/:id/review-ownership anonymous=401", ] `; + +exports[`route admission sentinel > admits actorless reads of team and workspace sessions but not private ones 1`] = ` +[ + "actorless-workspace=200", + "actorless-team=200", + "actorless-private=404", +] +`; + +exports[`route admission sentinel > audits one workspace Owner break-glass admission for a private session 1`] = ` +[ + "owner-role-break-glass=200:1", +] +`; + +exports[`route admission sentinel > conceals all team item routes from another team before any handler or DO call 1`] = ` +[ + "PATCH /sessions/:id/read-state other-team=404", + "DELETE /sessions/:id other-team=404", + "GET /sessions/:id/export other-team=404", + "GET /sessions/:id/sandbox-access other-team=404", + "GET /sessions/:id other-team=404", + "POST /sessions/:id/stop other-team=404", + "GET /sessions/:id/events other-team=404", + "GET /sessions/:id/artifacts other-team=404", + "GET /sessions/:id/participants other-team=404", + "GET /sessions/:id/participant-profiles other-team=404", + "GET /sessions/:id/messages other-team=404", + "POST /sessions/:id/pr other-team=404", + "GET /sessions/:id/tunnel-urls other-team=404", + "PATCH /sessions/:id/title other-team=404", + "POST /sessions/:id/archive other-team=404", + "POST /sessions/:id/unarchive other-team=404", + "PATCH /sessions/:id/budget other-team=404", + "POST /sessions/:id/ws-token other-team=404", + "POST /sessions/:id/prompt other-team=404", + "POST /sessions/:id/pull-requests/refresh other-team=404", + "POST /sessions/:id/media other-team=404", + "GET /sessions/:id/media/:artifactId other-team=404", + "POST /sessions/:id/attachments other-team=404", + "GET /sessions/:id/attachments/:attachmentId other-team=404", + "GET /sessions/:id/diff other-team=404", + "PUT /sessions/:id/diff other-team=404", + "POST /sessions/:id/diff/failure other-team=404", + "GET /sessions/:id/diff/:revisionId/files/:fileId other-team=404", + "POST /sessions/:id/diff/retry other-team=404", + "GET /sessions/:id/skills other-team=404", + "POST /sessions/:id/children other-team=404", + "GET /sessions/:id/children other-team=404", + "GET /sessions/:id/children/:childId other-team=404", + "POST /sessions/:id/children/:childId/cancel other-team=404", + "POST /sessions/:id/slack-notify other-team=404", +] +`; + +exports[`route admission sentinel > reports action denials for a same-team Viewer and admits a private collaborator 1`] = ` +[ + "PATCH /sessions/:id/read-state same-team-viewer=200", + "DELETE /sessions/:id same-team-viewer=403", + "GET /sessions/:id/sandbox-access same-team-viewer=403", + "GET /sessions/:id same-team-viewer=200", + "POST /sessions/:id/stop same-team-viewer=403", + "GET /sessions/:id/events same-team-viewer=200", + "GET /sessions/:id/artifacts same-team-viewer=200", + "GET /sessions/:id/participants same-team-viewer=200", + "GET /sessions/:id/participant-profiles same-team-viewer=200", + "GET /sessions/:id/messages same-team-viewer=200", + "POST /sessions/:id/pr same-team-viewer=403", + "GET /sessions/:id/tunnel-urls same-team-viewer=403", + "PATCH /sessions/:id/title same-team-viewer=403", + "POST /sessions/:id/archive same-team-viewer=403", + "POST /sessions/:id/unarchive same-team-viewer=403", + "PATCH /sessions/:id/budget same-team-viewer=403", + "POST /sessions/:id/ws-token same-team-viewer=200", + "POST /sessions/:id/prompt same-team-viewer=403", + "POST /sessions/:id/pull-requests/refresh same-team-viewer=403", + "POST /sessions/:id/media same-team-viewer=403", + "GET /sessions/:id/media/:artifactId same-team-viewer=200", + "POST /sessions/:id/attachments same-team-viewer=403", + "GET /sessions/:id/attachments/:attachmentId same-team-viewer=200", + "GET /sessions/:id/diff same-team-viewer=200", + "PUT /sessions/:id/diff same-team-viewer=403", + "POST /sessions/:id/diff/failure same-team-viewer=403", + "GET /sessions/:id/diff/:revisionId/files/:fileId same-team-viewer=200", + "POST /sessions/:id/diff/retry same-team-viewer=403", + "GET /sessions/:id/skills same-team-viewer=200", + "GET /sessions/:id/children same-team-viewer=200", + "GET /sessions/:id/children/:childId same-team-viewer=200", + "POST /sessions/:id/children/:childId/cancel same-team-viewer=403", + "POST /sessions/:id/slack-notify same-team-viewer=403", +] +`; + +exports[`route admission sentinel > reports action denials for a same-team Viewer and admits a private collaborator 2`] = ` +[ + "collaborator-on-private=200", +] +`; diff --git a/packages/control-plane/test/integration/helpers.ts b/packages/control-plane/test/integration/helpers.ts index a611d24f96..6288f66c0b 100644 --- a/packages/control-plane/test/integration/helpers.ts +++ b/packages/control-plane/test/integration/helpers.ts @@ -57,7 +57,7 @@ export const INTEGRATION_WEBSOCKET_TIMEOUT_MS = 2000; const TEST_BROWSER_USER_ID = "11111111111111111111111111111111"; const TEST_BROWSER_ACCOUNT_ID = "test-browser-account"; const TEST_BROWSER_PROVIDER_SUBJECT = "583231"; -type InitialUserRole = Exclude; +type InitialUserRole = BuiltInRoleKey; const DEFAULT_INITIAL_USER_ROLE = "owner" as const; const TEST_BROWSER_SESSION_ID = "test-browser-session"; const TEST_BROWSER_SESSION_TOKEN = "test-browser-session-token"; @@ -96,15 +96,21 @@ async function signCookieValue(value: string, secret: string): Promise { * web request must carry the same compound credential as production. Direct * service-auth tests intentionally build their own bare sig1 requests. */ -async function testBrowserSessionCookie(initialRole: InitialUserRole): Promise { +async function testBrowserSessionCookie( + initialRole: InitialUserRole, + as?: { userId: string; role: BuiltInRoleKey } +): Promise { const secret = env.BROWSER_AUTH_SECRET; if (!secret) throw new Error("BROWSER_AUTH_SECRET is not configured for integration tests"); const now = new Date(); const expiresAt = new Date(now.getTime() + 7 * 24 * 60 * 60 * 1000); const applicationTimestamp = now.getTime(); + const browserUserId = as?.userId ?? TEST_BROWSER_USER_ID; + const sessionId = as ? `test-browser-session-${browserUserId}` : TEST_BROWSER_SESSION_ID; + const sessionToken = as ? `test-browser-token-${browserUserId}` : TEST_BROWSER_SESSION_TOKEN; const existingUser = await env.DB.prepare("SELECT 1 FROM users WHERE id = ?") - .bind(TEST_BROWSER_USER_ID) + .bind(browserUserId) .first(); await env.DB.batch([ env.DB.prepare( @@ -112,11 +118,11 @@ async function testBrowserSessionCookie(initialRole: InitialUserRole): Promise { ), ]); + await env.DB.prepare( + `INSERT INTO sessions (id, repo_owner, repo_name, status, user_id, created_at, updated_at) + VALUES ('parent', 'acme', 'app', 'completed', ?, 1, 1)` + ) + .bind(user!.id) + .run(); + const response = await serviceFetch("https://cp.test/sessions/parent/children", { method: "POST", headers: { "Content-Type": "application/json" }, diff --git a/packages/control-plane/test/integration/route-admission-matrix.test.ts b/packages/control-plane/test/integration/route-admission-matrix.test.ts index 216c72be2b..be2a5e35ab 100644 --- a/packages/control-plane/test/integration/route-admission-matrix.test.ts +++ b/packages/control-plane/test/integration/route-admission-matrix.test.ts @@ -9,7 +9,7 @@ */ import { SELF, env } from "cloudflare:test"; -import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { buildServiceAuthHeaders } from "@open-inspect/shared/service-auth"; import { createExecutionContext } from "cloudflare:test"; import { cloudflareHost, createControlPlaneHttpHandler } from "../../src/cloudflare/http-host"; @@ -18,6 +18,8 @@ import { listRouteContracts, type RouteContract } from "../../src/routing/route- import { createCloudflareEnv } from "../../src/cloudflare/platform"; import { AutomationStore, type AutomationRow } from "../../src/db/automation-store"; import { TeamStore } from "../../src/db/teams"; +import { TeamMembershipStore } from "../../src/db/team-memberships"; +import { SessionCollaboratorStore } from "../../src/db/session-collaborators"; import { catalog } from "../../src/routes/catalog"; import { Hono } from "hono"; import { admit } from "../../src/routing/admit"; @@ -414,6 +416,11 @@ describe("route admission sentinel", { timeout: MATRIX_TIMEOUT_MS }, () => { ); } const handle = createControlPlaneHttpHandler([shadow]); + let teamSessionId = ""; + let privateSessionId = ""; + const OTHER_MEMBER = "33333333333333333333333333333333"; + const TEAM_VIEWER = "44444444444444444444444444444444"; + const COLLABORATOR = "55555555555555555555555555555555"; beforeAll(async () => { await cleanD1Tables(); @@ -430,6 +437,25 @@ describe("route admission sentinel", { timeout: MATRIX_TIMEOUT_MS }, () => { joinPolicy: "open", }) ).id; + for (const [userId, role] of [ + [OTHER_MEMBER, "member"], + [TEAM_VIEWER, "viewer"], + [COLLABORATOR, "member"], + ] as const) { + await serviceRequestHeaders(`${BASE}/me/authorization`, { as: { userId, role } }); + } + await new TeamMembershipStore(env.DB).add(fixtures.teamId, TEAM_VIEWER); + teamSessionId = await createReadySession(); + privateSessionId = await createReadySession(); + await env.DB.batch([ + env.DB.prepare( + "UPDATE sessions SET owner_team_id = ?, visibility = 'team' WHERE id = ?" + ).bind(fixtures.teamId, teamSessionId), + env.DB.prepare( + "UPDATE sessions SET owner_team_id = ?, visibility = 'private', user_id = ? WHERE id = ?" + ).bind(fixtures.teamId, OTHER_MEMBER, privateSessionId), + ]); + await new SessionCollaboratorStore(env.DB).add(privateSessionId, COLLABORATOR, OTHER_MEMBER); }, MATRIX_TIMEOUT_MS); afterAll(async () => { @@ -479,6 +505,7 @@ describe("route admission sentinel", { timeout: MATRIX_TIMEOUT_MS }, () => { : isAutomationRoute(route) ? fixtures.automationId : sessionId, + childId: fixtures.sandboxSessionId, })}`; const method = route.method; const expectReach = async ( @@ -550,6 +577,7 @@ describe("route admission sentinel", { timeout: MATRIX_TIMEOUT_MS }, () => { }); it("delivers path segments to handlers decoded exactly once", async () => { + await initSession({ sessionName: "abc/def", userId: BROWSER_USER_ID }); // Every production contract, admitted by its own policy, in front of a // handler that echoes the parameters Hono decoded. const echo = new Hono(); @@ -588,4 +616,131 @@ describe("route admission sentinel", { timeout: MATRIX_TIMEOUT_MS }, () => { await expect(response.json(), url).resolves.toEqual({ groups }); } }); + + it("conceals all team item routes from another team before any handler or DO call", async () => { + const get = vi.fn(() => { + throw new Error("Denied route reached the Durable Object"); + }); + const requestEnv = createCloudflareEnv({ + ...env, + TEAMS_ENFORCEMENT: "on", + SESSION: new Proxy(env.SESSION, { + get(target, property, receiver) { + if (property === "get") return get; + return Reflect.get(target, property, receiver); + }, + }), + }); + const observed: string[] = []; + for (const route of routes.filter( + (item) => isSessionRoute(item) && item.authorization.kind === "active-user" + )) { + const identity = `${route.method} ${route.path}`; + const url = `${BASE}${materialize(route, { id: teamSessionId, childId: fixtures.sandboxSessionId })}`; + const headers = await serviceRequestHeaders(url, { + method: route.method, + as: { userId: OTHER_MEMBER, role: "member" }, + }); + const response = await handle( + new Request(url, { method: route.method, headers }), + requestEnv, + createExecutionContext() + ); + observed.push(`${identity} other-team=${response.status}`); + expect(response.status, identity).toBe(404); + await expect(response.json(), identity).resolves.toEqual({ error: "Session not found" }); + } + expect(get).not.toHaveBeenCalled(); + expect(observed).toMatchSnapshot(); + }); + + it("reports action denials for a same-team Viewer and admits a private collaborator", async () => { + const observed: string[] = []; + for (const route of routes.filter( + (item) => isSessionRoute(item) && item.authorization.kind === "active-user" + )) { + if ( + route.path.endsWith("/export") || + (route.path.endsWith("/children") && route.method === "POST") + ) + continue; + const url = `${BASE}${materialize(route, { id: teamSessionId, childId: fixtures.sandboxSessionId })}`; + const headers = await serviceRequestHeaders(url, { + method: route.method, + as: { userId: TEAM_VIEWER, role: "viewer" }, + }); + const response = await handle( + new Request(url, { method: route.method, headers }), + createCloudflareEnv({ ...env, TEAMS_ENFORCEMENT: "on" }), + createExecutionContext() + ); + const expected = + route.authorization.kind === "active-user" && + route.authorization.allOf.every( + (entry) => entry.kind !== "session" || entry.action === "read" + ) + ? 200 + : 403; + observed.push(`${route.method} ${route.path} same-team-viewer=${response.status}`); + expect(response.status, `${route.method} ${route.path}`).toBe(expected); + if (expected === 403) + await expect(response.json()).resolves.toMatchObject({ reason_code: "missing_permission" }); + } + expect(observed).toMatchSnapshot(); + const url = `${BASE}/sessions/${privateSessionId}/events`; + const headers = await serviceRequestHeaders(url, { + as: { userId: COLLABORATOR, role: "member" }, + }); + const response = await handle( + new Request(url, { headers }), + createCloudflareEnv({ ...env, TEAMS_ENFORCEMENT: "on" }), + createExecutionContext() + ); + expect(response.status).toBe(200); + expect([`collaborator-on-private=${response.status}`]).toMatchSnapshot(); + }); + + it("admits actorless reads of team and workspace sessions but not private ones", async () => { + const observed: string[] = []; + for (const id of [fixtures.readonlySessionId, teamSessionId, privateSessionId]) { + const url = `${BASE}/sessions/${id}/events`; + const response = await handle( + new Request(url, { + headers: await botHeaders(url, "GET", "slack-bot"), + }), + createCloudflareEnv({ ...env, TEAMS_ENFORCEMENT: "on" }), + createExecutionContext() + ); + observed.push( + `actorless-${id === fixtures.readonlySessionId ? "workspace" : id === teamSessionId ? "team" : "private"}=${response.status}` + ); + expect(response.status).toBe(id === privateSessionId ? 404 : 200); + } + expect(observed).toMatchSnapshot(); + }); + + it("audits one workspace Owner break-glass admission for a private session", async () => { + const url = `${BASE}/sessions/${privateSessionId}`; + const headers = await serviceRequestHeaders(url); + const response = await handle( + new Request(url, { headers }), + createCloudflareEnv({ ...env, TEAMS_ENFORCEMENT: "on" }), + createExecutionContext() + ); + expect(response.status).toBe(200); + const audits = await env.DB.prepare( + "SELECT resource_type, resource_id, team_id, actor_user_id_snapshot FROM authorization_audit_events WHERE action = 'session.private_break_glass'" + ).all(); + expect(audits.results).toEqual([ + { + resource_type: "session", + resource_id: privateSessionId, + team_id: fixtures.teamId, + actor_user_id_snapshot: BROWSER_USER_ID, + }, + ]); + expect([ + `owner-role-break-glass=${response.status}:${audits.results.length}`, + ]).toMatchSnapshot(); + }); }); diff --git a/packages/control-plane/test/integration/sandbox-events.test.ts b/packages/control-plane/test/integration/sandbox-events.test.ts index b9214db57e..d4329fad5a 100644 --- a/packages/control-plane/test/integration/sandbox-events.test.ts +++ b/packages/control-plane/test/integration/sandbox-events.test.ts @@ -8,6 +8,7 @@ import { queryDO, seedMessage, seedSandboxAuth, + waitForSandboxStatus, } from "./helpers"; import { runInSessionDO } from "./session-do-access"; @@ -452,6 +453,162 @@ describe("POST /internal/sandbox-event", () => { }); }); + it("projects a step that finishes after a stop settled the session", async () => { + const { stub, sessionName } = await initSession(); + const participants = await queryDO<{ id: string }>( + stub, + "SELECT id FROM participants WHERE user_id = 'user-1'" + ); + const msgId = "msg-late-step"; + await seedMessage(stub, { + id: msgId, + authorId: participants[0].id, + content: "Test prompt", + source: "web", + status: "processing", + createdAt: Date.now() - 1000, + startedAt: Date.now() - 500, + }); + const postEvent = (event: Record) => + stub.fetch("http://internal/internal/sandbox-event", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ sandboxId: "sb-1", messageId: msgId, ...event }), + }); + const stepFinish = (stepId: string, input: number) => + postEvent({ + type: "step_finish", + stepId, + timestamp: Date.now() / 1000, + tokens: { input, output: 40, reasoning: 5, cache: { read: 800, write: 60 } }, + }); + const index = new SessionIndexStore(env.DB); + + expect((await stepFinish("step-1", 100)).status).toBe(200); + // Stop settles the session before the sandbox has seen the stop command. + expect((await stub.fetch("http://internal/internal/stop", { method: "POST" })).status).toBe( + 200 + ); + await vi.waitFor(async () => { + expect(await index.get(sessionName)).toMatchObject({ status: "failed", inputTokens: 100 }); + }); + + // A step already in flight lands after the settle; its terminal is then a no-op. + expect((await stepFinish("step-2", 250)).status).toBe(200); + const res = await postEvent({ + type: "execution_complete", + success: false, + error: "Task was cancelled", + timestamp: Date.now() / 1000, + }); + expect(res.status).toBe(200); + + await vi.waitFor(async () => { + expect(await index.get(sessionName)).toMatchObject({ + inputTokens: 350, + outputTokens: 80, + reasoningTokens: 10, + cacheReadTokens: 1600, + cacheWriteTokens: 120, + }); + }); + }); + + it("projects a budget-stopped turn's steps while a queued prompt waits on the budget", async () => { + const { stub, sessionName } = await initSession({ sandboxSettings: { maxSessionCostUsd: 1 } }); + await waitForSandboxStatus(stub, "failed"); + const [{ id: authorId }] = await queryDO<{ id: string }>( + stub, + "SELECT id FROM participants WHERE user_id = 'user-1'" + ); + const stoppedId = "msg-budget-stopped"; + const queuedId = "msg-budget-queued"; + await seedMessage(stub, { + id: stoppedId, + authorId, + content: "Running prompt", + source: "web", + status: "processing", + createdAt: Date.now() - 1000, + startedAt: Date.now() - 500, + }); + await seedMessage(stub, { + id: queuedId, + authorId, + content: "Queued prompt", + source: "web", + status: "pending", + createdAt: Date.now() - 900, + }); + await queryDO(stub, "UPDATE session SET status = 'active'"); + const postEvent = (event: Record) => + stub.fetch("http://internal/internal/sandbox-event", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ sandboxId: "sb-1", messageId: stoppedId, ...event }), + }); + const stepFinish = (stepId: string, input: number, messageCostUsd: number) => + postEvent({ + type: "step_finish", + stepId, + timestamp: Date.now() / 1000, + tokens: { input, output: 40, reasoning: 5, cache: { read: 800, write: 60 } }, + messageCostUsd, + }); + const index = new SessionIndexStore(env.DB); + + expect((await stepFinish("step-1", 100, 0.4)).status).toBe(200); + // This step reaches the limit: the budget stops the turn, and the queued + // prompt keeps the session active but cannot dispatch. The stopped turn is + // projected anyway, since no settle is coming. + expect((await stepFinish("step-2", 200, 1.2)).status).toBe(200); + await vi.waitFor(async () => { + expect(await index.get(sessionName)).toMatchObject({ inputTokens: 300 }); + }); + // A step already in flight lands after the stop, then the sandbox's own terminal. + expect((await stepFinish("step-3", 250, 1.5)).status).toBe(200); + const terminal = await postEvent({ + type: "execution_complete", + success: false, + error: "Task was cancelled", + timestamp: Date.now() / 1000, + }); + expect(terminal.status).toBe(200); + + expect(await queryDO(stub, "SELECT status, budget_exhausted FROM session")).toEqual([ + { status: "active", budget_exhausted: 1 }, + ]); + expect(await queryDO(stub, "SELECT id, status FROM messages ORDER BY created_at")).toEqual([ + { id: stoppedId, status: "failed" }, + { id: queuedId, status: "pending" }, + ]); + const tokenKinds = [ + "inputTokens", + "outputTokens", + "reasoningTokens", + "cacheReadTokens", + "cacheWriteTokens", + ] as const; + const exported = await stub.fetch("http://internal/internal/trace-export?include=usage"); + expect(exported.status).toBe(200); + const { trace } = await exported.json<{ + trace: { usage: Array> }; + }>(); + const usageTotals = Object.fromEntries( + tokenKinds.map((kind) => [kind, trace.usage.reduce((sum, row) => sum + (row[kind] ?? 0), 0)]) + ); + expect(usageTotals).toEqual({ + inputTokens: 550, + outputTokens: 120, + reasoningTokens: 15, + cacheReadTokens: 2400, + cacheWriteTokens: 180, + }); + await vi.waitFor(async () => { + expect(await index.get(sessionName)).toMatchObject(usageTotals); + }); + }); + it("execution_complete with success=false marks message as failed", async () => { const { stub } = await initSession(); diff --git a/packages/control-plane/test/integration/session-access-routes.test.ts b/packages/control-plane/test/integration/session-access-routes.test.ts new file mode 100644 index 0000000000..d3cdc3f94b --- /dev/null +++ b/packages/control-plane/test/integration/session-access-routes.test.ts @@ -0,0 +1,503 @@ +import { createExecutionContext, env } from "cloudflare:test"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { TeamStore } from "../../src/db/teams"; +import { TeamMembershipStore } from "../../src/db/team-memberships"; +import { SessionCollaboratorStore } from "../../src/db/session-collaborators"; +import { cleanD1Tables } from "./cleanup"; +import { + initSession, + routeRequest, + seedActiveUser, + serviceRequestHeaders, + waitForSandboxStatus, +} from "./helpers"; + +const BASE = "https://test.local"; +const OWNER = "11111111111111111111111111111111"; +const MEMBER = "22222222222222222222222222222222"; +const CREATOR = "33333333333333333333333333333333"; + +async function fetchMode( + path: string, + mode: string, + options: { + method?: string; + as?: { userId: string; role: "owner" | "member" | "viewer" }; + body?: string; + service?: "linear-bot"; + } = {} +) { + const url = `${BASE}${path}`; + const method = options.method ?? "GET"; + return routeRequest( + new Request(url, { + method, + headers: await serviceRequestHeaders(url, { + method, + body: options.body, + as: options.as, + service: options.service, + }), + body: options.body, + }), + { ...env, TEAMS_ENFORCEMENT: mode }, + createExecutionContext() + ); +} + +async function auditRows(action: string) { + return ( + await env.DB.prepare( + "SELECT action, resource_type, resource_id, team_id, reason_code, actor_user_id_snapshot FROM authorization_audit_events WHERE action = ? ORDER BY occurred_at" + ) + .bind(action) + .all() + ).results; +} + +describe("HTTP session access by enforcement mode", () => { + beforeEach(async () => { + await cleanD1Tables(); + expect((await fetchMode("/me/authorization", "on")).status).toBe(200); + expect( + ( + await fetchMode("/me/authorization", "on", { + as: { userId: MEMBER, role: "member" }, + }) + ).status + ).toBe(200); + await seedActiveUser(CREATOR); + }); + + async function session(visibility: "team" | "private") { + const team = await new TeamStore(env.DB).create({ + slug: `access-${crypto.randomUUID()}`, + name: "Access Team", + joinPolicy: "invite_only", + }); + const { sessionName, stub } = await initSession({ userId: CREATOR }); + await waitForSandboxStatus(stub, "failed"); + await env.DB.prepare("UPDATE sessions SET owner_team_id = ?, visibility = ? WHERE id = ?") + .bind(team.id, visibility, sessionName) + .run(); + return { sessionName, team }; + } + + it("conceals a team session on read and token mint when enforcement is on", async () => { + const { sessionName, team } = await session("team"); + const as = { userId: MEMBER, role: "member" } as const; + const snapshot = await fetchMode(`/sessions/${sessionName}`, "on", { as }); + const token = await fetchMode(`/sessions/${sessionName}/ws-token`, "on", { + as, + method: "POST", + }); + expect(snapshot.status).toBe(404); + expect(await snapshot.json()).toEqual({ error: "Session not found" }); + expect(token.status).toBe(404); + const denied = await auditRows("authorization.request_denied"); + expect(denied.filter((row) => row.reason_code === "session_not_visible")).toHaveLength(2); + expect(denied.find((row) => row.reason_code === "session_not_visible")?.team_id).toBe(team.id); + }); + + it("defers the team and delete rules in shadow but records each would-be denial", async () => { + const { sessionName, team } = await session("team"); + const as = { userId: MEMBER, role: "member" } as const; + const snapshot = await fetchMode(`/sessions/${sessionName}`, "shadow", { as }); + expect(snapshot.status).toBe(200); + const shadowRows = (await auditRows("authorization.request_allowed")).filter( + (row) => typeof row.reason_code === "string" && row.reason_code.startsWith("shadow_denied:") + ); + expect(shadowRows).toMatchObject([ + { reason_code: "shadow_denied:not_member", team_id: team.id }, + ]); + + await new TeamMembershipStore(env.DB).add(team.id, MEMBER); + const denied = await fetchMode(`/sessions/${sessionName}`, "on", { as }); + expect(denied.status).toBe(200); + const deletion = await fetchMode(`/sessions/${sessionName}`, "on", { as, method: "DELETE" }); + expect(deletion.status).toBe(403); + expect(await deletion.json()).toEqual({ + error: "Forbidden", + code: "session_action_denied", + reason_code: "not_owner_or_lead", + }); + await new TeamMembershipStore(env.DB).setRole(team.id, MEMBER, "lead"); + expect( + (await fetchMode(`/sessions/${sessionName}`, "on", { as, method: "DELETE" })).status + ).toBe(200); + }); + + it("writes one shadow audit row with the real status for reads and mutations", async () => { + const { sessionName } = await session("team"); + for (const [path, method, body] of [ + [`/sessions/${sessionName}`, "GET", undefined], + [`/sessions/${sessionName}/budget`, "PATCH", JSON.stringify({ maxCostUsd: 20 })], + ] as const) { + const response = await fetchMode(path, "shadow", { + method, + body, + as: { userId: MEMBER, role: "member" }, + }); + const rows = ( + await env.DB.prepare( + "SELECT reason_code, metadata_json FROM authorization_audit_events WHERE request_id = ? AND action = 'authorization.request_allowed'" + ) + .bind(response.headers.get("x-request-id")) + .all() + ).results; + expect(response.status).toBe(method === "GET" ? 200 : 403); + expect(rows).toHaveLength(1); + expect(rows[0].reason_code).toBe("shadow_denied:not_member"); + expect(JSON.parse(String(rows[0].metadata_json))).toMatchObject({ + httpStatus: response.status, + responseCode: "shadow_denied:not_member", + }); + } + }); + + it("records batch shadow denials in its one response-time audit row", async () => { + const { sessionName } = await session("team"); + await env.DB.batch([ + env.DB.prepare( + "INSERT INTO roles (id, key, name, normalized_name, is_system) VALUES ('role_batch_shadow', NULL, 'Batch Shadow', 'batch shadow', 0)" + ), + env.DB.prepare( + `INSERT INTO role_permissions (role_id, permission_id) + VALUES ('role_batch_shadow', 'sessions.bulk_archive'), + ('role_batch_shadow', 'sessions.read'), + ('role_batch_shadow', 'sessions.lifecycle')` + ), + env.DB.prepare( + "UPDATE user_role_assignments SET role_id = 'role_batch_shadow' WHERE user_id = ?" + ).bind(MEMBER), + ]); + const response = await fetchMode("/sessions/batch-archive", "shadow", { + method: "POST", + as: { userId: MEMBER, role: "member" }, + body: JSON.stringify({ sessionIds: [sessionName] }), + }); + const rows = ( + await env.DB.prepare( + "SELECT reason_code, metadata_json FROM authorization_audit_events WHERE request_id = ? AND action = 'authorization.request_allowed'" + ) + .bind(response.headers.get("x-request-id")) + .all() + ).results; + expect(rows).toHaveLength(1); + expect(rows[0].reason_code).toBe("shadow_denied:batch"); + expect(JSON.parse(String(rows[0].metadata_json))).toMatchObject({ + httpStatus: response.status, + responseCode: "shadow_denied:batch", + shadowDenials: [{ sessionId: sessionName, reason: "not_member" }], + }); + }); + + it("allows legacy deletion in shadow and off while shadow audits the ownership denial", async () => { + const as = { userId: MEMBER, role: "member" } as const; + const shadow = await session("team"); + await new TeamMembershipStore(env.DB).add(shadow.team.id, MEMBER); + expect( + (await fetchMode(`/sessions/${shadow.sessionName}`, "shadow", { method: "DELETE", as })) + .status + ).toBe(200); + expect( + (await auditRows("authorization.request_allowed")).filter( + (row) => row.reason_code === "shadow_denied:not_owner_or_lead" + ) + ).toHaveLength(1); + const off = await session("team"); + expect( + (await fetchMode(`/sessions/${off.sessionName}`, "off", { method: "DELETE", as })).status + ).toBe(200); + }); + + it("conceals another team's export even when the viewer holds sessions.export", async () => { + const { sessionName } = await session("team"); + await env.DB.batch([ + env.DB.prepare( + `INSERT INTO roles (id, key, name, normalized_name, is_system) + VALUES ('role_export_reader', NULL, 'Export Reader', 'export reader', 0)` + ), + env.DB.prepare( + `INSERT INTO role_permissions (role_id, permission_id) + VALUES ('role_export_reader', 'sessions.read'), + ('role_export_reader', 'sessions.export')` + ), + env.DB.prepare( + "UPDATE user_role_assignments SET role_id = 'role_export_reader' WHERE user_id = ?" + ).bind(MEMBER), + ]); + const response = await fetchMode(`/sessions/${sessionName}/export`, "on", { + as: { userId: MEMBER, role: "member" }, + }); + expect(response.status).toBe(404); + expect(await response.json()).toEqual({ error: "Session not found" }); + }); + + it("keeps private sessions concealed in all modes and audits Owner break-glass once per read", async () => { + const { sessionName, team } = await session("private"); + const as = { userId: MEMBER, role: "member" } as const; + await new TeamMembershipStore(env.DB).add(team.id, MEMBER); + for (const mode of ["off", "shadow", "on"] as const) { + expect((await fetchMode(`/sessions/${sessionName}`, mode, { as })).status).toBe(404); + } + expect((await fetchMode(`/sessions/${sessionName}`, "on")).status).toBe(200); + expect(await auditRows("session.private_break_glass")).toMatchObject([ + { + resource_type: "session", + resource_id: sessionName, + team_id: team.id, + actor_user_id_snapshot: OWNER, + }, + ]); + await new SessionCollaboratorStore(env.DB).add(sessionName, MEMBER, OWNER); + expect((await fetchMode(`/sessions/${sessionName}`, "on", { as })).status).toBe(200); + expect(await auditRows("session.private_break_glass")).toHaveLength(1); + }); + + it("does not query memberships in off mode", async () => { + const { sessionName } = await session("team"); + const list = vi.spyOn(TeamMembershipStore.prototype, "listForUser"); + try { + expect( + ( + await fetchMode(`/sessions/${sessionName}`, "off", { + as: { userId: MEMBER, role: "member" }, + }) + ).status + ).toBe(200); + expect(list).not.toHaveBeenCalled(); + } finally { + list.mockRestore(); + } + }); + + it("loads collaborators but not memberships for an off-mode private session", async () => { + const { sessionName } = await session("private"); + const members = vi.spyOn(TeamMembershipStore.prototype, "listForUser"); + const collaborators = vi.spyOn(SessionCollaboratorStore.prototype, "listUserIds"); + try { + expect((await fetchMode(`/sessions/${sessionName}`, "off")).status).toBe(200); + expect(members).not.toHaveBeenCalled(); + expect(collaborators).toHaveBeenCalledOnce(); + } finally { + members.mockRestore(); + collaborators.mockRestore(); + } + }); + + it("admits actorless Linear stop on non-private sessions in every mode, but hides private sessions", async () => { + const team = await session("team"); + const privateSession = await session("private"); + const workspace = await initSession({ userId: CREATOR }); + await waitForSandboxStatus(workspace.stub, "failed"); + for (const mode of ["off", "shadow", "on"] as const) { + for (const id of [team.sessionName, workspace.sessionName]) { + const response = await fetchMode(`/sessions/${id}/stop`, mode, { + method: "POST", + service: "linear-bot", + }); + expect(response.status).toBe(200); + } + expect( + ( + await fetchMode(`/sessions/${privateSession.sessionName}/stop`, mode, { + method: "POST", + service: "linear-bot", + }) + ).status + ).toBe(404); + } + expect( + (await auditRows("authorization.request_allowed")).filter( + (row) => row.reason_code === "shadow_denied:missing_permission" + ) + ).toEqual([]); + }); + + it("conceals an invisible child even when its parent is visible", async () => { + const parent = await initSession({ userId: CREATOR }); + await waitForSandboxStatus(parent.stub, "failed"); + const child = await session("team"); + await env.DB.prepare("UPDATE sessions SET parent_session_id = ? WHERE id = ?") + .bind(parent.sessionName, child.sessionName) + .run(); + const as = { userId: MEMBER, role: "member" } as const; + expect( + ( + await fetchMode(`/sessions/${parent.sessionName}/children/${child.sessionName}`, "on", { + as, + }) + ).status + ).toBe(404); + expect( + ( + await fetchMode( + `/sessions/${parent.sessionName}/children/${child.sessionName}/cancel`, + "on", + { as, method: "POST" } + ) + ).status + ).toBe(404); + expect( + (await auditRows("authorization.request_denied")).filter( + (row) => row.reason_code === "session_not_visible" + ) + ).toMatchObject([{ team_id: child.team.id }, { team_id: child.team.id }]); + }); + + it("lists only children visible in the selected enforcement mode", async () => { + const parent = await initSession({ userId: CREATOR }); + await waitForSandboxStatus(parent.stub, "failed"); + const workspace = await initSession({ userId: CREATOR }); + await waitForSandboxStatus(workspace.stub, "failed"); + const team = await session("team"); + const hidden = await session("private"); + for (const childId of [workspace.sessionName, team.sessionName, hidden.sessionName]) { + await env.DB.prepare("UPDATE sessions SET parent_session_id = ? WHERE id = ?") + .bind(parent.sessionName, childId) + .run(); + } + + for (const mode of ["off", "shadow", "on"] as const) { + const response = await fetchMode(`/sessions/${parent.sessionName}/children`, mode, { + as: { userId: MEMBER, role: "member" }, + }); + expect(response.status).toBe(200); + const body = (await response.json()) as { children: { id: string }[] }; + expect(body.children.map((child) => child.id).sort()).toEqual( + (mode === "on" ? [workspace.sessionName] : [workspace.sessionName, team.sessionName]).sort() + ); + } + expect( + (await auditRows("authorization.request_allowed")).filter( + (row) => row.reason_code === "shadow_denied:batch" + ) + ).toHaveLength(1); + }); + + it("audits both private reads when an Owner accesses a private child", async () => { + const parent = await session("private"); + const child = await session("private"); + await env.DB.prepare("UPDATE sessions SET parent_session_id = ? WHERE id = ?") + .bind(parent.sessionName, child.sessionName) + .run(); + await fetchMode(`/sessions/${parent.sessionName}/children/${child.sessionName}`, "on"); + expect( + (await auditRows("session.private_break_glass")).map((row) => row.resource_id).sort() + ).toEqual([parent.sessionName, child.sessionName].sort()); + }); + + it("audits a permitted private read before a later handler denial", async () => { + const { sessionName } = await session("private"); + const response = await fetchMode(`/sessions/${sessionName}/budget`, "off", { + method: "PATCH", + body: JSON.stringify({ maxCostUsd: 20 }), + }); + expect(response.status).toBe(403); + expect((await auditRows("session.private_break_glass")).map((row) => row.resource_id)).toEqual([ + sessionName, + ]); + }); + + it("responds 503 to an invalid mode for both item and batch routes", async () => { + expect((await fetchMode("/sessions/missing", "invalid")).status).toBe(503); + expect( + ( + await fetchMode("/sessions/batch-archive", "invalid", { + method: "POST", + body: JSON.stringify({ sessionIds: ["missing"] }), + }) + ).status + ).toBe(503); + }); + + it("skips hidden and action-denied batch targets independently", async () => { + const hidden = await session("private"); + const visible = await session("team"); + await new TeamMembershipStore(env.DB).add(visible.team.id, MEMBER); + await env.DB.batch([ + env.DB.prepare( + `INSERT INTO roles (id, key, name, normalized_name, is_system) + VALUES ('role_batch_viewer', NULL, 'Batch Viewer', 'batch viewer', 0)` + ), + env.DB.prepare( + `INSERT INTO role_permissions (role_id, permission_id) + VALUES ('role_batch_viewer', 'sessions.bulk_archive'), + ('role_batch_viewer', 'sessions.read')` + ), + env.DB.prepare( + "UPDATE user_role_assignments SET role_id = 'role_batch_viewer' WHERE user_id = ?" + ).bind(MEMBER), + ]); + const memberships = vi.spyOn(TeamMembershipStore.prototype, "listForUser"); + const response = await fetchMode("/sessions/batch-archive", "on", { + method: "POST", + as: { userId: MEMBER, role: "member" }, + body: JSON.stringify({ sessionIds: [hidden.sessionName, visible.sessionName] }), + }); + expect(memberships).toHaveBeenCalledOnce(); + memberships.mockRestore(); + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ + results: [], + skipped: [ + { sessionId: hidden.sessionName, reason: "not_found" }, + { sessionId: visible.sessionName, reason: "missing_permission" }, + ], + }); + }); + + it.each(["off", "shadow", "on"] as const)( + "preserves bulk-only custom-role archiving in %s mode", + async (mode) => { + const { sessionName, team } = await session("team"); + await new TeamMembershipStore(env.DB).add(team.id, MEMBER); + await env.DB.batch([ + env.DB.prepare( + `INSERT INTO roles (id, key, name, normalized_name, is_system) + VALUES ('role_bulk_only', NULL, 'Bulk Only', 'bulk only', 0)` + ), + env.DB.prepare( + `INSERT INTO role_permissions (role_id, permission_id) + VALUES ('role_bulk_only', 'sessions.bulk_archive'), + ('role_bulk_only', 'sessions.read')` + ), + env.DB.prepare( + "UPDATE user_role_assignments SET role_id = 'role_bulk_only' WHERE user_id = ?" + ).bind(MEMBER), + ]); + + const response = await fetchMode("/sessions/batch-archive", mode, { + method: "POST", + as: { userId: MEMBER, role: "member" }, + body: JSON.stringify({ sessionIds: [sessionName] }), + }); + expect(response.status).toBe(200); + expect(await response.json()).toEqual( + mode === "on" + ? { results: [], skipped: [{ sessionId: sessionName, reason: "missing_permission" }] } + : { results: [{ sessionId: sessionName, outcome: "archived" }], skipped: [] } + ); + if (mode === "shadow") { + expect( + (await auditRows("authorization.request_allowed")).some( + (row) => row.reason_code === "shadow_denied:batch" + ) + ).toBe(true); + } + } + ); + + it("lists, idempotently adds, and removes collaborators", async () => { + const { sessionName } = await session("private"); + const store = new SessionCollaboratorStore(env.DB); + expect(await store.add(sessionName, MEMBER, OWNER)).toBe(true); + expect(await store.add(sessionName, MEMBER, OWNER)).toBe(false); + expect(await store.listUserIds(sessionName)).toEqual([MEMBER]); + expect(await store.listForUser(MEMBER)).toEqual([sessionName]); + expect(await store.remove(sessionName, MEMBER)).toBe(true); + expect(await store.remove(sessionName, MEMBER)).toBe(false); + expect(await store.listUserIds(sessionName)).toEqual([]); + }); +}); diff --git a/packages/control-plane/test/integration/session-batch-archive.test.ts b/packages/control-plane/test/integration/session-batch-archive.test.ts index 946f2a9a0d..c2ab14ac79 100644 --- a/packages/control-plane/test/integration/session-batch-archive.test.ts +++ b/packages/control-plane/test/integration/session-batch-archive.test.ts @@ -46,7 +46,8 @@ describe("session batch archive", () => { const allowed = await member(); expect(allowed.status).toBe(200); expect(await allowed.json()).toEqual({ - results: [{ sessionId: "missing", outcome: "not_found" }], + results: [], + skipped: [{ sessionId: "missing", reason: "not_found" }], }); await env.DB.prepare("UPDATE user_role_assignments SET role_id = ? WHERE user_id = ?") .bind("role_builtin_member", USER_ID) @@ -91,13 +92,14 @@ describe("session batch archive", () => { expect(sessionBatchArchiveResponseSchema.parse(await response.json())).toEqual({ results: [ { sessionId: ready.sessionName, outcome: "archived" }, - { sessionId: "missing", outcome: "not_found" }, { sessionId: cancelled.sessionName, outcome: "skipped_cancelled" }, { sessionId: queued.sessionName, outcome: "skipped_queued_work" }, ], + skipped: [{ sessionId: "missing", reason: "not_found" }], }); expect(await (await post([ready.sessionName])).json()).toEqual({ results: [{ sessionId: ready.sessionName, outcome: "already_archived" }], + skipped: [], }); }); @@ -120,6 +122,7 @@ describe("session batch archive", () => { expect(response.status).toBe(200); expect(await response.json()).toEqual({ results: [{ sessionId: sessionName, outcome: "already_archived" }], + skipped: [], }); expect( await env.DB.prepare("SELECT status, updated_at FROM sessions WHERE id = ?") @@ -134,7 +137,8 @@ describe("session batch archive", () => { await waitForSandboxStatus(stub, "failed"); await env.DB.prepare("DELETE FROM sessions WHERE id = ?").bind(sessionName).run(); expect(await (await post([sessionName])).json()).toEqual({ - results: [{ sessionId: sessionName, outcome: "failed" }], + results: [], + skipped: [{ sessionId: sessionName, reason: "not_found" }], }); }); diff --git a/packages/control-plane/test/integration/team-stores.test.ts b/packages/control-plane/test/integration/team-stores.test.ts index c762af61b1..e88ce7f779 100644 --- a/packages/control-plane/test/integration/team-stores.test.ts +++ b/packages/control-plane/test/integration/team-stores.test.ts @@ -1,5 +1,6 @@ import { env } from "cloudflare:test"; import { beforeEach, describe, expect, it } from "vitest"; +import { EnvironmentStore, type EnvironmentRow } from "../../src/db/environments"; import { TeamSlugConflictError, TeamStore } from "../../src/db/teams"; import { TeamMembershipStore, @@ -10,6 +11,21 @@ import { cleanD1Tables } from "./cleanup"; beforeEach(cleanD1Tables); +function environmentRow(overrides: Partial): EnvironmentRow { + const now = Date.now(); + return { + id: "env_test", + name: "Test Environment", + description: null, + prebuild_enabled: 0, + channel_associations: null, + created_at: now, + updated_at: now, + owner_team_id: null, + ...overrides, + }; +} + describe("team and membership stores", () => { it("validates team rows and allows any team to be archived or restored", async () => { const store = new TeamStore(env.DB); @@ -43,6 +59,41 @@ describe("team and membership stores", () => { expect((await store.getById(team.id))?.defaultEnvironmentId).toBeNull(); }); + it("only accepts default environments owned by the team", async () => { + const teams = new TeamStore(env.DB); + const environments = new EnvironmentStore(env.DB); + const team = await teams.create({ + slug: "default-env", + name: "Default Env", + joinPolicy: "invite_only", + }); + const otherTeam = await teams.create({ + slug: "other-default-env", + name: "Other Default Env", + joinPolicy: "invite_only", + }); + await environments.create( + environmentRow({ id: "env_same_team", name: "Same Team", owner_team_id: team.id }), + [] + ); + await environments.create( + environmentRow({ id: "env_other_team", name: "Other Team", owner_team_id: otherTeam.id }), + [] + ); + + await expect( + teams.update(team.id, { defaultEnvironmentId: "env_same_team" }) + ).resolves.toMatchObject({ defaultEnvironmentId: "env_same_team" }); + + await expect(teams.update(team.id, { defaultEnvironmentId: "env_other_team" })).rejects.toThrow( + "Default environment must belong to the team" + ); + await expect(teams.update(team.id, { defaultEnvironmentId: "env_missing" })).rejects.toThrow( + "Default environment must belong to the team" + ); + expect((await teams.getById(team.id))?.defaultEnvironmentId).toBe("env_same_team"); + }); + it("reports duplicate slugs as a typed store conflict", async () => { const store = new TeamStore(env.DB); await store.create({ slug: "duplicate", name: "First", joinPolicy: "invite_only" }); diff --git a/packages/docs/content/docs/models/choosing-a-model.mdx b/packages/docs/content/docs/models/choosing-a-model.mdx index c7ab6f11c9..ce264a973e 100644 --- a/packages/docs/content/docs/models/choosing-a-model.mdx +++ b/packages/docs/content/docs/models/choosing-a-model.mdx @@ -4,7 +4,7 @@ description: Enable models for your workspace, pick a model and effort in the co audience: team-owner owner: platform status: published -lastReviewed: "2026-09-23" +lastReviewed: "2026-09-28" relatedCode: - packages/shared/src/models.ts - packages/shared/src/harnesses.ts @@ -62,6 +62,7 @@ Runs on both harnesses. Credential: `ANTHROPIC_API_KEY` as a global secret, or o | `anthropic/claude-sonnet-4-5` | Claude Sonnet 4.5 | high, max | max | | `anthropic/claude-sonnet-4-6` | Claude Sonnet 4.6 | low, medium, high, max | high | | `anthropic/claude-sonnet-5` | Claude Sonnet 5 | low, medium, high, xhigh, max | high | +| `anthropic/claude-sonnet-5-5` | Claude Sonnet 5.5 | low, medium, high, xhigh, max | high | | `anthropic/claude-opus-4-5` | Claude Opus 4.5 | high, max | max | | `anthropic/claude-opus-4-6` | Claude Opus 4.6 | low, medium, high, max | high | | `anthropic/claude-opus-4-7` | Claude Opus 4.7 | low, medium, high, xhigh, max | high | diff --git a/packages/modal-infra/pyproject.toml b/packages/modal-infra/pyproject.toml index fdb829fc6c..5e1c45bc05 100644 --- a/packages/modal-infra/pyproject.toml +++ b/packages/modal-infra/pyproject.toml @@ -5,7 +5,7 @@ description = "Modal sandbox infrastructure for Open-Inspect coding agent" requires-python = ">=3.12" dependencies = [ "open-inspect-sandbox-runtime", # sibling package, resolved via [tool.uv.sources] - "modal>=1.4.3", # Function.with_options() (per-call timeout override) requires >=1.4.3 + "modal==1.4.3", # Tested SDK; newer releases remove endpoint introspection used by tests "httpx>=0.27.0", "pydantic>=2.0", "fastapi>=0.110.0", diff --git a/packages/modal-infra/src/sandbox/launch.py b/packages/modal-infra/src/sandbox/launch.py new file mode 100644 index 0000000000..e3dde208ea --- /dev/null +++ b/packages/modal-infra/src/sandbox/launch.py @@ -0,0 +1,363 @@ +"""Translate a session launch into Modal image, environment, and resource arguments.""" + +import json +import secrets +import time +from dataclasses import dataclass +from typing import Any + +import modal + +from sandbox_runtime.constants import ( + CODE_SERVER_PORT_ENV_VAR, + DOCKER_ENABLED_ENV_VAR, + EXPECTED_TUNNEL_PORTS_ENV_VAR, + NOVNC_PORT_ENV_VAR, + SANDBOX_TIMEOUT_ENV_VAR, + TTYD_PROXY_PORT_ENV_VAR, + VNC_PASSWORD_ENV_VAR, + VNC_PASSWORD_MAX_BYTES, +) +from sandbox_runtime.log_config import get_logger +from sandbox_runtime.types import SandboxStatus + +from ..app import app +from ..app_config import APP_NAME +from ..images.base import base_image +from .launch_policy import ( + docker_allocation_name, + docker_allocation_tags, + docker_base_image, + docker_runtime_env, + launch_kwargs, + parse_launch, +) +from .models import SandboxConfig, SandboxHandle +from .tunnels import SandboxTunnels +from .vcs_env import inject_vcs_env_vars +from .vm_recovery import VMAllocationOutcome, VMServiceLaunch, find_owned_vm, owned_vm_tags_match + +_RESERVED_LAUNCH_ENV_VARS = { + "RESTORED_FROM_SNAPSHOT", + "FROM_REPO_IMAGE", + "REPO_IMAGE_SHA", + "IMAGE_BUILD_MODE", + "TERMINAL_ENABLED", + "AGENT_SLACK_NOTIFY_ENABLED", + "SESSION_CONFIG", + "CODE_SERVER_PASSWORD", + CODE_SERVER_PORT_ENV_VAR, + TTYD_PROXY_PORT_ENV_VAR, + EXPECTED_TUNNEL_PORTS_ENV_VAR, + VNC_PASSWORD_ENV_VAR, + NOVNC_PORT_ENV_VAR, + DOCKER_ENABLED_ENV_VAR, +} + +log = get_logger("manager") +ACCESS_PASSWORD_READ_TIMEOUT_SECONDS = 30 + + +class RepositoryImageUnavailableError(RuntimeError): + """The selected repository image no longer exists in Modal.""" + + +@dataclass(frozen=True) +class BaseImageSource: + pass + + +@dataclass(frozen=True) +class RepositoryImageSource: + image_id: str + sha: str | None + + +@dataclass(frozen=True) +class SnapshotImageSource: + image_id: str + clone_token: str | None + + +type SandboxImageSource = BaseImageSource | RepositoryImageSource | SnapshotImageSource + + +async def _create_sandbox( + create_kwargs: dict[str, Any], *, repository_image: bool +) -> modal.Sandbox: + """Only a missing repository image at create time is classified as unavailable.""" + try: + return await modal.Sandbox.create.aio( + "python", "-m", "sandbox_runtime.entrypoint", **create_kwargs + ) + except modal.exception.NotFoundError as e: + if repository_image: + raise RepositoryImageUnavailableError("repository image is unavailable") from e + raise + + +def _session_identity(session_config: Any) -> str: + if isinstance(session_config, dict): + session_id = session_config.get("session_id") + elif session_config is not None: + session_id = session_config.session_id + else: + session_id = None + return session_id if isinstance(session_id, str) else "" + + +@dataclass(frozen=True) +class SandboxLaunchSpec: + """Canonical launch configuration paired with one image source variant.""" + + config: SandboxConfig + source: SandboxImageSource + + +class SandboxLauncher: + """Own the common Modal launch path for base, repository, and snapshot images.""" + + @staticmethod + def _generate_code_server_password() -> str: + """Generate a random code-server password.""" + return secrets.token_urlsafe(16) + + @staticmethod + def _generate_vnc_password() -> str: + """Generate a random VNC password.""" + return secrets.token_urlsafe(VNC_PASSWORD_MAX_BYTES)[:VNC_PASSWORD_MAX_BYTES] + + async def launch(self, spec: SandboxLaunchSpec) -> SandboxHandle: + """Launch a Modal sandbox from a normalized create or restore specification.""" + config = spec.config + has_repository = bool(config.repo_owner) + sandbox_id = config.sandbox_id + if not sandbox_id: + sandbox_name = ( + f"{config.repo_owner}-{config.repo_name}" if has_repository else "no-repository" + ) + sandbox_id = f"sandbox-{sandbox_name}-{int(time.time() * 1000)}" + + docker = parse_launch(config.sandbox_backend, config.settings) + env_vars = { + key: value + for key, value in (config.user_env_vars or {}).items() + if key not in _RESERVED_LAUNCH_ENV_VARS + } + env_vars.update( + { + "PYTHONUNBUFFERED": "1", + "SANDBOX_ID": sandbox_id, + "CONTROL_PLANE_URL": config.control_plane_url, + "SANDBOX_AUTH_TOKEN": config.sandbox_auth_token, + SANDBOX_TIMEOUT_ENV_VAR: str(config.timeout_seconds), + "REPO_OWNER": config.repo_owner or "", + "REPO_NAME": config.repo_name or "", + **docker_runtime_env(docker), + } + ) + + clone_token: str | None = None + include_github_cli_aliases = False + snapshot_id: str | None = None + if isinstance(spec.source, BaseImageSource): + image = docker_base_image() if docker.enabled else base_image + elif isinstance(spec.source, RepositoryImageSource): + try: + image = modal.Image.from_id(spec.source.image_id) + except modal.exception.NotFoundError as e: + raise RepositoryImageUnavailableError("repository image is unavailable") from e + env_vars["FROM_REPO_IMAGE"] = "true" + env_vars["REPO_IMAGE_SHA"] = spec.source.sha or "" + else: + image = modal.Image.from_id(spec.source.image_id) + env_vars["RESTORED_FROM_SNAPSHOT"] = "true" + clone_token = spec.source.clone_token + include_github_cli_aliases = True + snapshot_id = spec.source.image_id + + if config.session_config is not None: + env_vars["SESSION_CONFIG"] = ( + json.dumps(config.session_config) + if isinstance(config.session_config, dict) + else config.session_config.model_dump_json() + ) + + inject_vcs_env_vars( + env_vars, + clone_token=clone_token if has_repository else None, + include_github_cli_aliases=include_github_cli_aliases, + ) + + code_server_password: str | None = None + if config.code_server_enabled: + code_server_password = self._generate_code_server_password() + env_vars["CODE_SERVER_PASSWORD"] = code_server_password + + vnc_password: str | None = None + if config.vnc_enabled: + vnc_password = self._generate_vnc_password() + env_vars[VNC_PASSWORD_ENV_VAR] = vnc_password + + if config.agent_slack_notify_enabled: + env_vars["AGENT_SLACK_NOTIFY_ENABLED"] = "true" + + tunnels = SandboxTunnels( + code_server_enabled=config.code_server_enabled, + vnc_enabled=config.vnc_enabled, + settings=config.settings, + ) + env_vars.update(tunnels.environment) + + # A fresh handle avoids Modal caching the ID of a deleted/recreated secret. + llm_secrets = modal.Secret.from_name("llm-api-keys") + await llm_secrets.hydrate.aio() + create_kwargs: dict[str, Any] = { + "image": image, + "app": app, + "secrets": [llm_secrets], + "timeout": config.timeout_seconds, + "workdir": "/workspace", + "env": env_vars, + **launch_kwargs(docker), + } + if tunnels.exposed_ports: + create_kwargs["encrypted_ports"] = tunnels.exposed_ports + + repository_image = isinstance(spec.source, RepositoryImageSource) + if docker.enabled: + sandbox, adopted = await self._launch_docker_sandbox( + session_id=_session_identity(config.session_config), + sandbox_id=sandbox_id, + retire_sandbox_id=config.retire_sandbox_id, + create_kwargs=create_kwargs, + repository_image=repository_image, + launch_deadline_at_ms=config.launch_deadline_at_ms, + service_launch=VMServiceLaunch.from_tunnels(tunnels), + ) + if adopted: + passwords = await self._read_access_passwords( + sandbox, + code_server_enabled=config.code_server_enabled, + vnc_enabled=config.vnc_enabled, + ) + code_server_password = passwords.get("CODE_SERVER_PASSWORD") + vnc_password = passwords.get(VNC_PASSWORD_ENV_VAR) + else: + sandbox = await _create_sandbox(create_kwargs, repository_image=repository_image) + modal_object_id = sandbox.object_id + urls = await tunnels.resolve(sandbox, sandbox_id) + + return SandboxHandle( + sandbox_id=sandbox_id, + modal_sandbox=sandbox, + status=SandboxStatus.WARMING, + created_at=time.time(), + snapshot_id=snapshot_id, + modal_object_id=modal_object_id, + code_server_url=urls.code_server_url, + code_server_password=code_server_password, + vnc_url=urls.vnc_url, + vnc_password=vnc_password, + ttyd_url=urls.ttyd_url, + tunnel_urls=urls.tunnel_urls, + sandbox_backend=docker.backend, + ) + + async def _launch_docker_sandbox( + self, + *, + session_id: str, + sandbox_id: str, + retire_sandbox_id: str | None, + create_kwargs: dict[str, Any], + repository_image: bool, + service_launch: VMServiceLaunch, + launch_deadline_at_ms: int | None = None, + ) -> tuple[modal.Sandbox, bool]: + """Create a named VM or adopt only the allocation owned by this generation.""" + if retire_sandbox_id: + await self._retire_docker_allocation(session_id, retire_sandbox_id) + name = docker_allocation_name(session_id) + tags = docker_allocation_tags(session_id, sandbox_id) + existing = await self._find_owned_docker_allocation(name, tags) + if existing is None: + if launch_deadline_at_ms is not None and time.time() * 1000 >= launch_deadline_at_ms: + raise VMAllocationOutcome("window_closed", "VM launch deadline expired") + try: + sandbox = await _create_sandbox( + {**create_kwargs, "name": name, "tags": {**tags, **service_launch.tags()}}, + repository_image=repository_image, + ) + return sandbox, False + except modal.exception.AlreadyExistsError as e: + existing = await self._find_owned_docker_allocation(name, tags) + if existing is None: + raise VMAllocationOutcome( + "race_pending", "VM allocation is not yet visible" + ) from e + log.info( + "sandbox.docker_allocation_adopted", + sandbox_id=sandbox_id, + modal_object_id=existing.object_id, + ) + return existing, True + + @staticmethod + async def _read_access_passwords( + sandbox: modal.Sandbox, *, code_server_enabled: bool, vnc_enabled: bool + ) -> dict[str, str]: + """Recover enabled service credentials from the owned VM launch environment.""" + keys = [] + if code_server_enabled: + keys.append("CODE_SERVER_PASSWORD") + if vnc_enabled: + keys.append(VNC_PASSWORD_ENV_VAR) + if not keys: + return {} + process = await sandbox.exec.aio( + "python", + "-I", + "-c", + "import json, os, sys; print(json.dumps({k: os.environ.get(k) for k in sys.argv[1:]}))", + *keys, + timeout=ACCESS_PASSWORD_READ_TIMEOUT_SECONDS, + ) + output = await process.stdout.read.aio() + if await process.wait.aio() != 0: + raise RuntimeError("Could not recover adopted sandbox access credentials") + try: + passwords = json.loads(output) + except ValueError: + raise RuntimeError("Could not recover adopted sandbox access credentials") from None + if not isinstance(passwords, dict) or any( + not isinstance(passwords.get(key), str) or not passwords[key] for key in keys + ): + raise RuntimeError("Could not recover adopted sandbox access credentials") + return {key: passwords[key] for key in keys} + + @staticmethod + async def _find_owned_docker_allocation( + name: str, tags: dict[str, str] + ) -> modal.Sandbox | None: + found = await find_owned_vm(name, tags) + return found[0] if found else None + + async def _retire_docker_allocation(self, session_id: str, sandbox_id: str) -> None: + """Terminate a prior named VM only when its ownership tags match.""" + name = docker_allocation_name(session_id) + try: + sandbox = await modal.Sandbox.from_name.aio(APP_NAME, name) + except modal.exception.NotFoundError: + return + if not owned_vm_tags_match( + await sandbox.get_tags.aio(), docker_allocation_tags(session_id, sandbox_id) + ): + log.warn("sandbox.docker_allocation_retire_mismatch", sandbox_id=sandbox_id) + return + await sandbox.terminate.aio(wait=True) + log.info( + "sandbox.docker_allocation_retired", + sandbox_id=sandbox_id, + modal_object_id=sandbox.object_id, + ) diff --git a/packages/modal-infra/src/sandbox/manager.py b/packages/modal-infra/src/sandbox/manager.py index 30e044eb8e..0f1c4dd46a 100644 --- a/packages/modal-infra/src/sandbox/manager.py +++ b/packages/modal-infra/src/sandbox/manager.py @@ -1,18 +1,6 @@ -""" -Sandbox lifecycle management for Open-Inspect. +"""Provider lifecycle operations for Open-Inspect session sandboxes.""" -This module handles: -- Creating sandboxes from filesystem snapshots -- Taking snapshots for session persistence - -Updated: 2026-01-15 to fix Sandbox.create API -""" - -import asyncio -import json -import secrets import time -from dataclasses import dataclass from typing import Any import modal @@ -38,44 +26,67 @@ from sandbox_runtime.log_config import get_logger from sandbox_runtime.types import SandboxStatus, SessionConfig -from ..app import app from ..app_config import APP_NAME -from ..images.base import base_image +from .launch import ( + ACCESS_PASSWORD_READ_TIMEOUT_SECONDS, + BaseImageSource, + RepositoryImageSource, + RepositoryImageUnavailableError, + SandboxImageSource, + SandboxLauncher, + SandboxLaunchSpec, + SnapshotImageSource, +) from .launch_policy import ( PENDING_VM_REFERENCE_PREFIX, ModalBackend, docker_allocation_name, docker_allocation_tags, - docker_base_image, - docker_runtime_env, - launch_kwargs, - parse_launch, parse_pending_vm_reference, ) -from .vcs_env import inject_vcs_env_vars +from .models import DEFAULT_VNC_ENABLED, SandboxConfig, SandboxHandle +from .tunnels import MAX_TUNNEL_PORTS +from .vm_recovery import ( + VMAllocationOutcome, + find_owned_vm, + owned_vm_tags_match, + recover_vm_access, +) + +# Preserve the existing public imports after moving their implementations. +__all__ = [ + "ACCESS_PASSWORD_READ_TIMEOUT_SECONDS", + "APP_NAME", + "CODE_SERVER_PORT", + "CODE_SERVER_PORT_ENV_VAR", + "CONTROL_TIMEOUT_SECONDS", + "DEFAULT_SANDBOX_TIMEOUT_SECONDS", + "DEFAULT_VNC_ENABLED", + "DOCKER_ENABLED_ENV_VAR", + "EXPECTED_TUNNEL_PORTS_ENV_VAR", + "MAX_TUNNEL_PORTS", + "NOVNC_PORT", + "NOVNC_PORT_ENV_VAR", + "PENDING_VM_REFERENCE_PREFIX", + "SANDBOX_TIMEOUT_ENV_VAR", + "SNAPSHOT_FILESYSTEM_TIMEOUT_SECONDS", + "TTYD_PROXY_PORT", + "TTYD_PROXY_PORT_ENV_VAR", + "TUNNEL_ENV_FILE_PATH", + "TUNNEL_ENV_SANDBOX_ID_KEY", + "VNC_PASSWORD_ENV_VAR", + "VNC_PASSWORD_MAX_BYTES", + "VNC_PORT", + "RepositoryImageUnavailableError", + "SandboxConfig", + "SandboxHandle", + "SandboxManager", + "VMAllocationOutcome", +] log = get_logger("manager") SNAPSHOT_FILESYSTEM_TIMEOUT_SECONDS = 300 -ACCESS_PASSWORD_READ_TIMEOUT_SECONDS = 30 -MAX_TUNNEL_PORTS = 10 -DEFAULT_VNC_ENABLED = False -_RESERVED_LAUNCH_ENV_VARS = { - "RESTORED_FROM_SNAPSHOT", - "FROM_REPO_IMAGE", - "REPO_IMAGE_SHA", - "IMAGE_BUILD_MODE", - "TERMINAL_ENABLED", - "AGENT_SLACK_NOTIFY_ENABLED", - "SESSION_CONFIG", - VNC_PASSWORD_ENV_VAR, - NOVNC_PORT_ENV_VAR, - DOCKER_ENABLED_ENV_VAR, -} - - -class RepositoryImageUnavailableError(RuntimeError): - """The selected repository image no longer exists in Modal.""" class PendingVMReferenceNotVisible(RuntimeError): @@ -90,574 +101,13 @@ def _has_repository(repo_owner: str | None, repo_name: str | None) -> bool: return has_owner -async def _create_sandbox( - create_kwargs: dict[str, Any], *, repository_image: bool -) -> modal.Sandbox: - """The one `Sandbox.create` call; only its own NotFound means the image is gone.""" - try: - return await modal.Sandbox.create.aio( - "python", - "-m", - "sandbox_runtime.entrypoint", - **create_kwargs, - ) - except modal.exception.NotFoundError as e: - if repository_image: - raise RepositoryImageUnavailableError("repository image is unavailable") from e - raise - - -def _session_identity(session_config: SessionConfig | dict[str, Any] | None) -> str: - """The control-plane session id carried in the launch's session config.""" - if isinstance(session_config, dict): - session_id = session_config.get("session_id") - elif session_config is not None: - session_id = session_config.session_id - else: - session_id = None - return session_id if isinstance(session_id, str) else "" - - -@dataclass -class SandboxConfig: - """Configuration for creating a sandbox.""" - - repo_owner: str | None - repo_name: str | None - sandbox_backend: ModalBackend = "modal" - sandbox_id: str | None = None # Expected sandbox ID from control plane - session_config: SessionConfig | dict[str, Any] | None = None - control_plane_url: str = "" - sandbox_auth_token: str = "" - timeout_seconds: int = DEFAULT_SANDBOX_TIMEOUT_SECONDS - user_env_vars: dict[str, str] | None = None # User-provided env vars (repo secrets) - repo_image_id: str | None = None # Pre-built repo image ID from provider - repo_image_sha: str | None = None # Git SHA the repo image was built from - code_server_enabled: bool = False # Whether to start code-server in the sandbox - vnc_enabled: bool = DEFAULT_VNC_ENABLED # Whether to start the browser-accessible VNC desktop - agent_slack_notify_enabled: bool = ( - False # Whether to install the agent-initiated slack-notify tool - ) - settings: dict[str, Any] | None = ( - None # Sandbox settings (tunnelPorts, etc.) from control plane - ) - # A previous generation's sandbox id whose Docker VM may still be running - # after an ambiguous create (the control plane lost the response). Only - # Docker launches act on it; the named allocation is retired if owned. - retire_sandbox_id: str | None = None - launch_deadline_at_ms: int | None = None - - -@dataclass -class SandboxHandle: - """Handle to a sandbox.""" - - sandbox_id: str - modal_sandbox: modal.Sandbox - status: SandboxStatus - created_at: float - snapshot_id: str | None = None - modal_object_id: str | None = None # Modal's internal sandbox ID for API calls - code_server_url: str | None = None - code_server_password: str | None = None - vnc_url: str | None = None - vnc_password: str | None = None - ttyd_url: str | None = None # proxy tunnel URL (not ttyd directly) - tunnel_urls: dict[int, str] | None = None # port -> tunnel URL mapping for extra ports - sandbox_backend: ModalBackend = "modal" - - -@dataclass(frozen=True) -class _BaseImageSource: - pass - - -@dataclass(frozen=True) -class _RepositoryImageSource: - image_id: str - sha: str | None - - -@dataclass(frozen=True) -class _SnapshotImageSource: - image_id: str - clone_token: str | None - - -type _SandboxImageSource = _BaseImageSource | _RepositoryImageSource | _SnapshotImageSource - - -@dataclass(frozen=True) -class _SandboxLaunchSpec: - """Canonical launch configuration paired with one image source variant.""" - - config: SandboxConfig - source: _SandboxImageSource - - class SandboxManager: - """ - Manages sandbox lifecycle for Open-Inspect sessions. + """Normalize create/restore requests and manage existing provider sandboxes. - Responsibilities: - - Create sandboxes from snapshots or fresh images - - Take snapshots for session persistence + Launch translation and networking are owned by provider-local collaborators. + Session readiness and checkpoint/shutdown policy remain in the control plane. """ - @staticmethod - def _generate_code_server_password() -> str: - """Generate a random code-server password.""" - return secrets.token_urlsafe(16) - - @staticmethod - def _generate_vnc_password() -> str: - """Generate a random VNC password.""" - return secrets.token_urlsafe(VNC_PASSWORD_MAX_BYTES)[:VNC_PASSWORD_MAX_BYTES] - - @staticmethod - async def _resolve_tunnels( - sandbox: modal.Sandbox, - sandbox_id: str, - ports: list[int], - retries: int = 3, - backoff: float = 1.0, - ) -> dict[int, str]: - """Resolve tunnel URLs for the given ports from Modal, retrying on failure.""" - resolved: dict[int, str] = {} - for attempt in range(retries): - try: - loop = asyncio.get_running_loop() - tunnels = await loop.run_in_executor(None, sandbox.tunnels) - for port in ports: - if port in tunnels and port not in resolved: - resolved[port] = tunnels[port].url - log.info( - "tunnel.resolved", - sandbox_id=sandbox_id, - port=port, - url=tunnels[port].url, - ) - if len(resolved) == len(ports): - return resolved - except Exception as e: - log.warn( - "tunnel.resolve_error", - sandbox_id=sandbox_id, - attempt=attempt + 1, - retries=retries, - error=type(e).__name__, - exc=e, - ) - if attempt < retries - 1: - await asyncio.sleep(backoff * (attempt + 1)) - return resolved - - @staticmethod - def _validate_ports(raw: list) -> list[int]: - """Validate and sanitize tunnel ports: must be int, 1-65535, max MAX_TUNNEL_PORTS.""" - ports: list[int] = [] - for p in raw: - if isinstance(p, int) and 1 <= p <= 65535: - ports.append(p) - if len(ports) >= MAX_TUNNEL_PORTS: - break - return ports - - @staticmethod - def _resolve_service_ports(settings: dict[str, Any] | None) -> tuple[int, int, int]: - """Return effective (code_server_port, novnc_port, ttyd_proxy_port) from settings. - - Falls back to the service defaults when unset or invalid. The control - plane validates these before they reach here. - """ - s = settings or {} - - def coerce(value: Any, default: int) -> int: - if isinstance(value, int) and not isinstance(value, bool) and 1 <= value <= 65535: - return value - return default - - return ( - coerce(s.get("codeServerPort"), CODE_SERVER_PORT), - coerce(s.get("vncPort"), NOVNC_PORT), - coerce(s.get("terminalPort"), TTYD_PROXY_PORT), - ) - - @staticmethod - def _collect_exposed_ports( - code_server_enabled: bool, - vnc_enabled: bool, - terminal_enabled: bool, - settings: dict[str, Any] | None, - code_server_port: int, - novnc_port: int, - ttyd_proxy_port: int, - ) -> tuple[list[int], list[int]]: - """Return (all_exposed_ports, extra_tunnel_ports) from settings and feature flags.""" - # Raw VNC is localhost-only and must never be exposed, including as a - # user-configured extra tunnel. - reserved: set[int] = {VNC_PORT} - exposed: list[int] = [] - if code_server_enabled: - exposed.append(code_server_port) - reserved.add(code_server_port) - if vnc_enabled: - exposed.append(novnc_port) - reserved.add(novnc_port) - if terminal_enabled: - exposed.append(ttyd_proxy_port) - reserved.add(ttyd_proxy_port) - - raw_ports = (settings or {}).get("tunnelPorts", []) - tunnel_ports = SandboxManager._validate_ports(raw_ports) if raw_ports else [] - # Remove reserved ports from tunnel_ports to avoid duplicates - tunnel_ports = [p for p in tunnel_ports if p not in reserved] - exposed.extend(tunnel_ports) - return exposed, tunnel_ports - - @staticmethod - async def _resolve_and_setup_tunnels( - sandbox: modal.Sandbox, - sandbox_id: str, - code_server_enabled: bool, - vnc_enabled: bool, - terminal_enabled: bool, - extra_ports: list[int], - code_server_port: int, - novnc_port: int, - ttyd_proxy_port: int, - ) -> tuple[str | None, str | None, str | None, dict[int, str] | None]: - """Return (code_server_url, vnc_url, ttyd_url, extra_urls).""" - all_ports: list[int] = [] - if code_server_enabled: - all_ports.append(code_server_port) - if vnc_enabled: - all_ports.append(novnc_port) - if terminal_enabled: - all_ports.append(ttyd_proxy_port) - all_ports.extend(extra_ports) - - if not all_ports: - return None, None, None, None - - resolved = await SandboxManager._resolve_tunnels(sandbox, sandbox_id, all_ports) - - # Only pull a service port out of the resolved map when that service owns - # it. Otherwise a user's own port (e.g. 8080 with code-server disabled) - # would be misrouted to code_server_url and dropped from the tunnel map. - code_server_url = resolved.pop(code_server_port, None) if code_server_enabled else None - vnc_url = resolved.pop(novnc_port, None) if vnc_enabled else None - ttyd_url = resolved.pop(ttyd_proxy_port, None) if terminal_enabled else None - extra_urls = resolved if resolved else None - - if extra_urls: - await SandboxManager._write_tunnel_env_file(sandbox, sandbox_id, extra_urls) - - return code_server_url, vnc_url, ttyd_url, extra_urls - - @staticmethod - async def _write_tunnel_env_file( - sandbox: modal.Sandbox, - sandbox_id: str, - tunnel_urls: dict[int, str], - ) -> None: - """Write tunnel URLs to TUNNEL_ENV_FILE_PATH as a dotenv file. - - The first line tags the file with this sandbox's ID so the supervisor's - stale-file cleanup can tell a fresh write (this write can land before - the entrypoint runs) from a snapshot/image leftover. - - Failures are logged but do not block sandbox creation; URLs are also - returned to the control plane via the SandboxHandle. - """ - lines = [f"{TUNNEL_ENV_SANDBOX_ID_KEY}={sandbox_id}"] - lines += [f"TUNNEL_{port}={url}" for port, url in sorted(tunnel_urls.items())] - content = "\n".join(lines) + "\n" - try: - await sandbox.filesystem.write_text.aio(content, TUNNEL_ENV_FILE_PATH) - log.info( - "tunnel.urls_written", - sandbox_id=sandbox_id, - path=TUNNEL_ENV_FILE_PATH, - ports=list(tunnel_urls.keys()), - ) - except Exception as e: - log.warn( - "tunnel.urls_write_failed", - sandbox_id=sandbox_id, - path=TUNNEL_ENV_FILE_PATH, - exc=e, - ) - - async def _launch_sandbox(self, spec: _SandboxLaunchSpec) -> SandboxHandle: - """Launch a Modal sandbox from a normalized create or restore specification.""" - config = spec.config - has_repository = bool(config.repo_owner) - sandbox_id = config.sandbox_id - if not sandbox_id: - sandbox_name = ( - f"{config.repo_owner}-{config.repo_name}" if has_repository else "no-repository" - ) - sandbox_id = f"sandbox-{sandbox_name}-{int(time.time() * 1000)}" - - docker = parse_launch(config.sandbox_backend, config.settings) - env_vars = { - key: value - for key, value in (config.user_env_vars or {}).items() - if key not in _RESERVED_LAUNCH_ENV_VARS - } - env_vars.update( - { - "PYTHONUNBUFFERED": "1", - "SANDBOX_ID": sandbox_id, - "CONTROL_PLANE_URL": config.control_plane_url, - "SANDBOX_AUTH_TOKEN": config.sandbox_auth_token, - SANDBOX_TIMEOUT_ENV_VAR: str(config.timeout_seconds), - "REPO_OWNER": config.repo_owner or "", - "REPO_NAME": config.repo_name or "", - **docker_runtime_env(docker), - } - ) - - clone_token: str | None = None - include_github_cli_aliases = False - snapshot_id: str | None = None - if isinstance(spec.source, _BaseImageSource): - image = docker_base_image() if docker.enabled else base_image - elif isinstance(spec.source, _RepositoryImageSource): - image = modal.Image.from_id(spec.source.image_id) - env_vars["FROM_REPO_IMAGE"] = "true" - env_vars["REPO_IMAGE_SHA"] = spec.source.sha or "" - else: - image = modal.Image.from_id(spec.source.image_id) - env_vars["RESTORED_FROM_SNAPSHOT"] = "true" - clone_token = spec.source.clone_token - include_github_cli_aliases = True - snapshot_id = spec.source.image_id - - if config.session_config is not None: - env_vars["SESSION_CONFIG"] = ( - json.dumps(config.session_config) - if isinstance(config.session_config, dict) - else config.session_config.model_dump_json() - ) - - inject_vcs_env_vars( - env_vars, - clone_token=clone_token if has_repository else None, - include_github_cli_aliases=include_github_cli_aliases, - ) - - code_server_password: str | None = None - if config.code_server_enabled: - code_server_password = self._generate_code_server_password() - env_vars["CODE_SERVER_PASSWORD"] = code_server_password - - vnc_password: str | None = None - if config.vnc_enabled: - vnc_password = self._generate_vnc_password() - env_vars[VNC_PASSWORD_ENV_VAR] = vnc_password - - terminal_enabled = bool((config.settings or {}).get("terminalEnabled", False)) - if terminal_enabled: - env_vars["TERMINAL_ENABLED"] = "true" - if config.agent_slack_notify_enabled: - env_vars["AGENT_SLACK_NOTIFY_ENABLED"] = "true" - - code_server_port, novnc_port, ttyd_proxy_port = self._resolve_service_ports(config.settings) - if config.code_server_enabled: - env_vars[CODE_SERVER_PORT_ENV_VAR] = str(code_server_port) - if config.vnc_enabled: - env_vars[NOVNC_PORT_ENV_VAR] = str(novnc_port) - if terminal_enabled: - env_vars[TTYD_PROXY_PORT_ENV_VAR] = str(ttyd_proxy_port) - - exposed_ports, tunnel_ports = self._collect_exposed_ports( - config.code_server_enabled, - config.vnc_enabled, - terminal_enabled, - config.settings, - code_server_port, - novnc_port, - ttyd_proxy_port, - ) - if tunnel_ports: - env_vars[EXPECTED_TUNNEL_PORTS_ENV_VAR] = ",".join(str(p) for p in tunnel_ports) - - # from_name handles cache their resolved ID; use a fresh handle on every - # launch so a deleted and recreated secret can be resolved again. - llm_secrets = modal.Secret.from_name("llm-api-keys") - await llm_secrets.hydrate.aio() - - create_kwargs: dict[str, Any] = { - "image": image, - "app": app, - "secrets": [llm_secrets], - "timeout": config.timeout_seconds, - "workdir": "/workspace", - "env": env_vars, - **launch_kwargs(docker), - } - if exposed_ports: - create_kwargs["encrypted_ports"] = exposed_ports - - repository_image = isinstance(spec.source, _RepositoryImageSource) - if docker.enabled: - sandbox, adopted = await self._launch_docker_sandbox( - session_id=_session_identity(config.session_config), - sandbox_id=sandbox_id, - retire_sandbox_id=config.retire_sandbox_id, - create_kwargs=create_kwargs, - repository_image=repository_image, - launch_deadline_at_ms=config.launch_deadline_at_ms, - ) - if adopted: - passwords = await self._read_access_passwords( - sandbox, - code_server_enabled=config.code_server_enabled, - vnc_enabled=config.vnc_enabled, - ) - code_server_password = passwords.get("CODE_SERVER_PASSWORD") - vnc_password = passwords.get(VNC_PASSWORD_ENV_VAR) - else: - sandbox = await _create_sandbox(create_kwargs, repository_image=repository_image) - modal_object_id = sandbox.object_id - ( - code_server_url, - vnc_url, - ttyd_url, - extra_tunnel_urls, - ) = await self._resolve_and_setup_tunnels( - sandbox, - sandbox_id, - config.code_server_enabled, - config.vnc_enabled, - terminal_enabled, - tunnel_ports, - code_server_port, - novnc_port, - ttyd_proxy_port, - ) - - return SandboxHandle( - sandbox_id=sandbox_id, - modal_sandbox=sandbox, - status=SandboxStatus.WARMING, - created_at=time.time(), - snapshot_id=snapshot_id, - modal_object_id=modal_object_id, - code_server_url=code_server_url, - code_server_password=code_server_password, - vnc_url=vnc_url, - vnc_password=vnc_password, - ttyd_url=ttyd_url, - tunnel_urls=extra_tunnel_urls, - sandbox_backend=docker.backend, - ) - - async def _launch_docker_sandbox( - self, - *, - session_id: str, - sandbox_id: str, - retire_sandbox_id: str | None, - create_kwargs: dict[str, Any], - repository_image: bool, - launch_deadline_at_ms: int | None = None, - ) -> tuple[modal.Sandbox, bool]: - """Create a Docker VM under a deterministic name, adopting an existing one. - - VM creation can outlive the control plane's HTTP request. One name per - session serializes generations at Modal even when a predecessor lookup - misses an in-flight create. Only matching generation tags permit adoption. - """ - if retire_sandbox_id: - await self._retire_docker_allocation(session_id, retire_sandbox_id) - name = docker_allocation_name(session_id) - tags = docker_allocation_tags(session_id, sandbox_id) - existing = await self._find_owned_docker_allocation(name, tags) - if existing is None: - if launch_deadline_at_ms is not None and time.time() * 1000 >= launch_deadline_at_ms: - raise RuntimeError("VM launch deadline expired") - try: - sandbox = await _create_sandbox( - {**create_kwargs, "name": name, "tags": tags}, - repository_image=repository_image, - ) - return sandbox, False - except modal.exception.AlreadyExistsError: - existing = await self._find_owned_docker_allocation(name, tags) - if existing is None: - raise - log.info( - "sandbox.docker_allocation_adopted", - sandbox_id=sandbox_id, - modal_object_id=existing.object_id, - ) - return existing, True - - @staticmethod - async def _read_access_passwords( - sandbox: modal.Sandbox, *, code_server_enabled: bool, vnc_enabled: bool - ) -> dict[str, str]: - """Recover only enabled service credentials from the owned VM's launch environment.""" - keys = [] - if code_server_enabled: - keys.append("CODE_SERVER_PASSWORD") - if vnc_enabled: - keys.append(VNC_PASSWORD_ENV_VAR) - if not keys: - return {} - process = await sandbox.exec.aio( - "python", - "-I", - "-c", - "import json, os, sys; print(json.dumps({k: os.environ.get(k) for k in sys.argv[1:]}))", - *keys, - timeout=ACCESS_PASSWORD_READ_TIMEOUT_SECONDS, - ) - output = await process.stdout.read.aio() - if await process.wait.aio() != 0: - raise RuntimeError("Could not recover adopted sandbox access credentials") - try: - passwords = json.loads(output) - except ValueError: - raise RuntimeError("Could not recover adopted sandbox access credentials") from None - if not isinstance(passwords, dict) or any( - not isinstance(passwords.get(key), str) or not passwords[key] for key in keys - ): - raise RuntimeError("Could not recover adopted sandbox access credentials") - return {key: passwords[key] for key in keys} - - @staticmethod - async def _find_owned_docker_allocation( - name: str, tags: dict[str, str] - ) -> modal.Sandbox | None: - try: - sandbox = await modal.Sandbox.from_name.aio(APP_NAME, name) - except modal.exception.NotFoundError: - return None - if await sandbox.get_tags.aio() != tags: - raise RuntimeError("Docker sandbox allocation ownership mismatch") - return sandbox - - async def _retire_docker_allocation(self, session_id: str, sandbox_id: str) -> None: - """Terminate a prior generation's named VM, only when its ownership tags match.""" - name = docker_allocation_name(session_id) - try: - sandbox = await modal.Sandbox.from_name.aio(APP_NAME, name) - except modal.exception.NotFoundError: - return - if await sandbox.get_tags.aio() != docker_allocation_tags(session_id, sandbox_id): - log.warn("sandbox.docker_allocation_retire_mismatch", sandbox_id=sandbox_id) - return - await sandbox.terminate.aio(wait=True) - log.info( - "sandbox.docker_allocation_retired", - sandbox_id=sandbox_id, - modal_object_id=sandbox.object_id, - ) - async def create_sandbox( self, config: SandboxConfig, @@ -667,7 +117,7 @@ async def create_sandbox( Creates from the pre-built repo image when one is provided, otherwise from the base image. Snapshot restores go through - restore_sandbox, not this path. + restore_from_snapshot, not this path. Args: config: Sandbox configuration including repo info and session config @@ -679,14 +129,14 @@ async def create_sandbox( _has_repository(config.repo_owner, config.repo_name) if config.repo_image_id: - source: _SandboxImageSource = _RepositoryImageSource( + source: SandboxImageSource = RepositoryImageSource( image_id=config.repo_image_id, sha=config.repo_image_sha, ) else: - source = _BaseImageSource() + source = BaseImageSource() - handle = await self._launch_sandbox(_SandboxLaunchSpec(config=config, source=source)) + handle = await SandboxLauncher().launch(SandboxLaunchSpec(config=config, source=source)) duration_ms = int((time.time() - start_time) * 1000) log.info( @@ -800,14 +250,18 @@ async def get_sandbox_by_id(self, sandbox_id: str) -> SandboxHandle | None: APP_NAME, docker_allocation_name(identity[0]) ) except modal.exception.NotFoundError: - raise PendingVMReferenceNotVisible("VM launch identity is not yet visible") + raise PendingVMReferenceNotVisible( + "VM launch identity is not yet visible" + ) from None else: try: modal_sandbox = await modal.Sandbox.from_id.aio(sandbox_id) except modal.exception.NotFoundError: return None tags = await modal_sandbox.get_tags.aio() - if identity is not None and tags != docker_allocation_tags(*identity): + if identity is not None and not owned_vm_tags_match( + tags, docker_allocation_tags(*identity) + ): raise PendingVMReferenceNotVisible("Docker sandbox allocation ownership mismatch") backend = tags.get("openinspect_backend", "modal") if backend not in ("modal", "modal-vm"): @@ -817,10 +271,36 @@ async def get_sandbox_by_id(self, sandbox_id: str) -> SandboxHandle | None: sandbox_id=sandbox_id, modal_object_id=modal_sandbox.object_id, modal_sandbox=modal_sandbox, - status=SandboxStatus.READY, # Assume ready if we can retrieve it + status=SandboxStatus.READY, created_at=time.time(), ) + async def resolve_vm_sandbox(self, session_id: str, sandbox_id: str) -> SandboxHandle: + """Recover only the running generation's identity and versioned access metadata.""" + found = await find_owned_vm( + docker_allocation_name(session_id), docker_allocation_tags(session_id, sandbox_id) + ) + if found is None: + raise VMAllocationOutcome("not_visible", "VM allocation is not visible") + sandbox, tags = found + access = await recover_vm_access( + sandbox, sandbox_id, tags, SandboxLauncher._read_access_passwords + ) + return SandboxHandle( + sandbox_id=sandbox_id, + modal_sandbox=sandbox, + status=SandboxStatus.WARMING, + created_at=time.time(), + modal_object_id=sandbox.object_id, + code_server_url=access.code_server_url, + code_server_password=access.code_server_password, + vnc_url=access.vnc_url, + vnc_password=access.vnc_password, + ttyd_url=access.ttyd_url, + tunnel_urls=access.tunnel_urls, + sandbox_backend="modal-vm", + ) + async def restore_from_snapshot( self, snapshot_image_id: str, @@ -874,8 +354,8 @@ async def restore_from_snapshot( # so the gh CLI keeps working on snapshots predating the gh wrapper. # Host scoping remains common with fresh creates. These compatibility # credentials are explicitly requested only by the restore path. - handle = await self._launch_sandbox( - _SandboxLaunchSpec( + handle = await SandboxLauncher().launch( + SandboxLaunchSpec( config=SandboxConfig( repo_owner=repo_owner, repo_name=repo_name, @@ -888,12 +368,12 @@ async def restore_from_snapshot( code_server_enabled=code_server_enabled, vnc_enabled=vnc_enabled, agent_slack_notify_enabled=agent_slack_notify_enabled, - retire_sandbox_id=retire_sandbox_id, settings=settings, + retire_sandbox_id=retire_sandbox_id, sandbox_backend=sandbox_backend, launch_deadline_at_ms=launch_deadline_at_ms, ), - source=_SnapshotImageSource( + source=SnapshotImageSource( image_id=snapshot_image_id, clone_token=clone_token, ), diff --git a/packages/modal-infra/src/sandbox/models.py b/packages/modal-infra/src/sandbox/models.py new file mode 100644 index 0000000000..61125ecaa1 --- /dev/null +++ b/packages/modal-infra/src/sandbox/models.py @@ -0,0 +1,59 @@ +"""Configuration and handles shared by Modal sandbox operations.""" + +from dataclasses import dataclass +from typing import Any + +import modal + +from sandbox_runtime.constants import DEFAULT_SANDBOX_TIMEOUT_SECONDS +from sandbox_runtime.types import SandboxStatus, SessionConfig + +from .launch_policy import ModalBackend + +DEFAULT_VNC_ENABLED = False + + +@dataclass +class SandboxConfig: + """Configuration for creating a sandbox.""" + + repo_owner: str | None + repo_name: str | None + sandbox_id: str | None = None # Expected sandbox ID from control plane + session_config: SessionConfig | dict[str, Any] | None = None + control_plane_url: str = "" + sandbox_auth_token: str = "" + timeout_seconds: int = DEFAULT_SANDBOX_TIMEOUT_SECONDS + user_env_vars: dict[str, str] | None = None # User-provided env vars (repo secrets) + repo_image_id: str | None = None # Pre-built repo image ID from provider + repo_image_sha: str | None = None # Git SHA the repo image was built from + code_server_enabled: bool = False # Whether to start code-server in the sandbox + vnc_enabled: bool = DEFAULT_VNC_ENABLED # Whether to start the browser-accessible VNC desktop + agent_slack_notify_enabled: bool = ( + False # Whether to install the agent-initiated slack-notify tool + ) + settings: dict[str, Any] | None = ( + None # Sandbox settings (tunnelPorts, etc.) from control plane + ) + sandbox_backend: ModalBackend = "modal" + retire_sandbox_id: str | None = None + launch_deadline_at_ms: int | None = None + + +@dataclass +class SandboxHandle: + """Handle to a sandbox.""" + + sandbox_id: str + modal_sandbox: modal.Sandbox + status: SandboxStatus + created_at: float + snapshot_id: str | None = None + modal_object_id: str | None = None # Modal's internal sandbox ID for API calls + code_server_url: str | None = None + code_server_password: str | None = None + vnc_url: str | None = None + vnc_password: str | None = None + ttyd_url: str | None = None # proxy tunnel URL (not ttyd directly) + tunnel_urls: dict[int, str] | None = None # port -> tunnel URL mapping for extra ports + sandbox_backend: ModalBackend = "modal" diff --git a/packages/modal-infra/src/sandbox/tunnels.py b/packages/modal-infra/src/sandbox/tunnels.py new file mode 100644 index 0000000000..831854eefb --- /dev/null +++ b/packages/modal-infra/src/sandbox/tunnels.py @@ -0,0 +1,221 @@ +"""Service port ownership and best-effort Modal tunnel publication.""" + +import asyncio +from typing import Any, NamedTuple + +import modal + +from sandbox_runtime.constants import ( + CODE_SERVER_PORT, + CODE_SERVER_PORT_ENV_VAR, + EXPECTED_TUNNEL_PORTS_ENV_VAR, + NOVNC_PORT, + NOVNC_PORT_ENV_VAR, + TTYD_PROXY_PORT, + TTYD_PROXY_PORT_ENV_VAR, + TUNNEL_ENV_FILE_PATH, + TUNNEL_ENV_SANDBOX_ID_KEY, + VNC_PORT, +) +from sandbox_runtime.log_config import get_logger + +# Preserve the logger name used by existing launch/tunnel dashboards. +log = get_logger("manager") +MAX_TUNNEL_PORTS = 10 +DEFAULT_TUNNEL_RESOLUTION_RETRIES = 3 +DEFAULT_TUNNEL_RESOLUTION_BACKOFF_SECONDS = 1.0 + + +class TunnelUrls(NamedTuple): + """Resolved service URLs and any user-requested tunnels.""" + + code_server_url: str | None = None + vnc_url: str | None = None + ttyd_url: str | None = None + tunnel_urls: dict[int, str] | None = None + + +class SandboxTunnels: + """Keep exposed ports, runtime environment, and URL routing in agreement. + + Service ownership is resolved once for a launch. Disabled service ports + remain available as user tunnels; raw VNC is never an extra tunnel. + """ + + def __init__( + self, + *, + code_server_enabled: bool = False, + vnc_enabled: bool = False, + settings: dict[str, Any] | None = None, + ) -> None: + settings = settings or {} + self.service_ports = self._resolve_service_ports(settings) + self.service_enabled = ( + code_server_enabled, + vnc_enabled, + bool(settings.get("terminalEnabled", False)), + ) + code_server_port, novnc_port, ttyd_proxy_port = self.service_ports + self._code_server_port = code_server_port if code_server_enabled else None + self._novnc_port = novnc_port if vnc_enabled else None + self._ttyd_proxy_port = ttyd_proxy_port if self.service_enabled[2] else None + service_ports = [ + port + for port in (self._code_server_port, self._novnc_port, self._ttyd_proxy_port) + if port is not None + ] + reserved = {VNC_PORT, *service_ports} + raw_ports = settings.get("tunnelPorts", []) + self.extra_ports = ( + [port for port in self._validate_ports(raw_ports) if port not in reserved] + if raw_ports + else [] + ) + self.exposed_ports = service_ports + self.extra_ports + + @property + def environment(self) -> dict[str, str]: + """Runtime settings derived from the same ports used for exposure.""" + env: dict[str, str] = {} + if self._code_server_port is not None: + env[CODE_SERVER_PORT_ENV_VAR] = str(self._code_server_port) + if self._novnc_port is not None: + env[NOVNC_PORT_ENV_VAR] = str(self._novnc_port) + if self._ttyd_proxy_port is not None: + env["TERMINAL_ENABLED"] = "true" + env[TTYD_PROXY_PORT_ENV_VAR] = str(self._ttyd_proxy_port) + if self.extra_ports: + env[EXPECTED_TUNNEL_PORTS_ENV_VAR] = ",".join(str(p) for p in self.extra_ports) + return env + + async def resolve( + self, sandbox: modal.Sandbox, sandbox_id: str, *, write_env_file: bool = True + ) -> TunnelUrls: + """Resolve URLs and publish extras; partial resolution/write failures are non-fatal.""" + if not self.exposed_ports: + return TunnelUrls() + + resolved = await self._resolve_tunnels(sandbox, sandbox_id, self.exposed_ports) + # A disabled service does not own its default port: leave it in extras. + code_server_url = ( + resolved.pop(self._code_server_port, None) + if self._code_server_port is not None + else None + ) + vnc_url = resolved.pop(self._novnc_port, None) if self._novnc_port is not None else None + ttyd_url = ( + resolved.pop(self._ttyd_proxy_port, None) if self._ttyd_proxy_port is not None else None + ) + extra_urls = resolved or None + if extra_urls and write_env_file: + await self._write_tunnel_env_file(sandbox, sandbox_id, extra_urls) + return TunnelUrls( + code_server_url=code_server_url, + vnc_url=vnc_url, + ttyd_url=ttyd_url, + tunnel_urls=extra_urls, + ) + + @staticmethod + async def _resolve_tunnels( + sandbox: modal.Sandbox, + sandbox_id: str, + ports: list[int], + retries: int = DEFAULT_TUNNEL_RESOLUTION_RETRIES, + backoff_seconds: float = DEFAULT_TUNNEL_RESOLUTION_BACKOFF_SECONDS, + ) -> dict[int, str]: + """Resolve tunnel URLs for the given ports from Modal, retrying on failure.""" + resolved: dict[int, str] = {} + for attempt in range(retries): + try: + loop = asyncio.get_running_loop() + tunnels = await loop.run_in_executor(None, sandbox.tunnels) + for port in ports: + if port in tunnels and port not in resolved: + resolved[port] = tunnels[port].url + log.info( + "tunnel.resolved", + sandbox_id=sandbox_id, + port=port, + url=tunnels[port].url, + ) + if len(resolved) == len(ports): + return resolved + except Exception as e: + log.warn( + "tunnel.resolve_error", + sandbox_id=sandbox_id, + attempt=attempt + 1, + retries=retries, + error=type(e).__name__, + exc=e, + ) + if attempt < retries - 1: + await asyncio.sleep(backoff_seconds * (attempt + 1)) + return resolved + + @staticmethod + def _validate_ports(raw: list[Any]) -> list[int]: + """Validate and sanitize tunnel ports: must be int, 1-65535, max MAX_TUNNEL_PORTS.""" + ports: list[int] = [] + for p in raw: + if isinstance(p, int) and not isinstance(p, bool) and 1 <= p <= 65535: + ports.append(p) + if len(ports) >= MAX_TUNNEL_PORTS: + break + return ports + + @staticmethod + def _resolve_service_ports(settings: dict[str, Any] | None) -> tuple[int, int, int]: + """Return effective (code_server_port, novnc_port, ttyd_proxy_port) from settings. + + Falls back to the service defaults when unset or invalid. The control + plane validates these before they reach here. + """ + s = settings or {} + + def coerce(value: Any, default: int) -> int: + if isinstance(value, int) and not isinstance(value, bool) and 1 <= value <= 65535: + return value + return default + + return ( + coerce(s.get("codeServerPort"), CODE_SERVER_PORT), + coerce(s.get("vncPort"), NOVNC_PORT), + coerce(s.get("terminalPort"), TTYD_PROXY_PORT), + ) + + @staticmethod + async def _write_tunnel_env_file( + sandbox: modal.Sandbox, + sandbox_id: str, + tunnel_urls: dict[int, str], + ) -> None: + """Write tunnel URLs to TUNNEL_ENV_FILE_PATH as a dotenv file. + + The first line tags the file with this sandbox's ID so the supervisor's + stale-file cleanup can tell a fresh write (this write can land before + the entrypoint runs) from a snapshot/image leftover. + + Failures are logged but do not block sandbox creation; URLs are also + returned to the control plane via the SandboxHandle. + """ + lines = [f"{TUNNEL_ENV_SANDBOX_ID_KEY}={sandbox_id}"] + lines += [f"TUNNEL_{port}={url}" for port, url in sorted(tunnel_urls.items())] + content = "\n".join(lines) + "\n" + try: + await sandbox.filesystem.write_text.aio(content, TUNNEL_ENV_FILE_PATH) + log.info( + "tunnel.urls_written", + sandbox_id=sandbox_id, + path=TUNNEL_ENV_FILE_PATH, + ports=list(tunnel_urls.keys()), + ) + except Exception as e: + log.warn( + "tunnel.urls_write_failed", + sandbox_id=sandbox_id, + path=TUNNEL_ENV_FILE_PATH, + exc=e, + ) diff --git a/packages/modal-infra/src/sandbox/vm_recovery.py b/packages/modal-infra/src/sandbox/vm_recovery.py new file mode 100644 index 0000000000..3f6759df15 --- /dev/null +++ b/packages/modal-infra/src/sandbox/vm_recovery.py @@ -0,0 +1,157 @@ +"""Lookup-only recovery of named VM allocations and their provider-owned access metadata.""" + +from collections.abc import Awaitable, Callable +from dataclasses import dataclass +from typing import Literal + +import modal + +from sandbox_runtime.constants import VNC_PASSWORD_ENV_VAR + +from ..app_config import APP_NAME +from .tunnels import SandboxTunnels + +VM_LAUNCH_TAG = "openinspect_vm_launch" +VM_PORTS_TAG = "openinspect_vm_ports" +_METADATA_TAGS = {VM_LAUNCH_TAG, VM_PORTS_TAG} + +type VMAllocationDetail = Literal[ + "not_visible", "other_generation", "window_closed", "race_pending" +] + + +class VMAllocationOutcome(RuntimeError): + """A known named-VM lookup or launch outcome, distinct from provider failures.""" + + def __init__(self, detail: VMAllocationDetail, message: str): + super().__init__(message) + self.detail = detail + + +@dataclass(frozen=True) +class VMServiceLaunch: + code_server_enabled: bool + vnc_enabled: bool + terminal_enabled: bool + code_server_port: int + novnc_port: int + ttyd_proxy_port: int + tunnel_ports: list[int] + + @classmethod + def from_tunnels(cls, tunnels: SandboxTunnels) -> "VMServiceLaunch": + return cls(*tunnels.service_enabled, *tunnels.service_ports, tunnels.extra_ports) + + def tags(self) -> dict[str, str]: + # Two short, tag-safe values keep even ten five-digit extra ports within Modal's limit. + flags = "".join( + "1" if enabled else "0" + for enabled in (self.code_server_enabled, self.vnc_enabled, self.terminal_enabled) + ) + return { + VM_LAUNCH_TAG: ( + f"1-{flags}-{self.code_server_port}-{self.novnc_port}-{self.ttyd_proxy_port}" + ), + VM_PORTS_TAG: "-".join(map(str, self.tunnel_ports)) or "none", + } + + +def owned_vm_tags_match(actual: dict[str, str], expected: dict[str, str]) -> bool: + """Keep exact generation ownership, allowing only the two launch metadata tags.""" + return all(actual.get(key) == value for key, value in expected.items()) and ( + actual.keys() <= expected.keys() | _METADATA_TAGS + ) + + +def parse_vm_service_launch(tags: dict[str, str]) -> VMServiceLaunch | None: + """Absent, unsupported, or incomplete metadata grants no access (including on legacy VMs).""" + launch = tags.get(VM_LAUNCH_TAG, "").split("-") + raw_ports = tags.get(VM_PORTS_TAG) + if len(launch) != 5 or launch[0] != "1" or len(launch[1]) != 3: + return None + if any(flag not in "01" for flag in launch[1]) or raw_ports is None: + return None + + def port(value: str) -> int | None: + if not value.isascii() or not value.isdecimal() or len(value) > 5: + return None + number = int(value) + return number if 1 <= number <= 65535 else None + + service_ports = [port(value) for value in launch[2:]] + extras = [] if raw_ports == "none" else [port(value) for value in raw_ports.split("-")] + if None in service_ports or None in extras or len(extras) > 10: + return None + return VMServiceLaunch( + *(flag == "1" for flag in launch[1]), + *service_ports, + extras, + ) + + +async def find_owned_vm( + name: str, expected_tags: dict[str, str] +) -> tuple[modal.Sandbox, dict[str, str]] | None: + try: + sandbox = await modal.Sandbox.from_name.aio(APP_NAME, name) + except modal.exception.NotFoundError: + return None + tags = await sandbox.get_tags.aio() + if not owned_vm_tags_match(tags, expected_tags): + raise VMAllocationOutcome( + "other_generation", "Docker sandbox allocation ownership mismatch" + ) + return sandbox, tags + + +@dataclass +class VMAccess: + code_server_url: str | None = None + code_server_password: str | None = None + vnc_url: str | None = None + vnc_password: str | None = None + ttyd_url: str | None = None + tunnel_urls: dict[int, str] | None = None + + +async def recover_vm_access( + sandbox: modal.Sandbox, + sandbox_id: str, + tags: dict[str, str], + read_passwords: Callable[..., Awaitable[dict[str, str]]], +) -> VMAccess: + launch = parse_vm_service_launch(tags) + if launch is None: + return VMAccess() + passwords = await read_passwords( + sandbox, + code_server_enabled=launch.code_server_enabled, + vnc_enabled=launch.vnc_enabled, + ) + tunnels = SandboxTunnels( + code_server_enabled=launch.code_server_enabled, + vnc_enabled=launch.vnc_enabled, + settings={ + "terminalEnabled": launch.terminal_enabled, + "codeServerPort": launch.code_server_port, + "vncPort": launch.novnc_port, + "terminalPort": launch.ttyd_proxy_port, + "tunnelPorts": launch.tunnel_ports, + }, + ) + urls = await tunnels.resolve(sandbox, sandbox_id, write_env_file=False) + if ( + (launch.code_server_enabled and not urls.code_server_url) + or (launch.vnc_enabled and not urls.vnc_url) + or (launch.terminal_enabled and not urls.ttyd_url) + or any(not (urls.tunnel_urls or {}).get(port) for port in launch.tunnel_ports) + ): + raise VMAllocationOutcome("race_pending", "VM allocation tunnels are not yet visible") + return VMAccess( + code_server_url=urls.code_server_url, + code_server_password=passwords.get("CODE_SERVER_PASSWORD"), + vnc_url=urls.vnc_url, + vnc_password=passwords.get(VNC_PASSWORD_ENV_VAR), + ttyd_url=urls.ttyd_url, + tunnel_urls=urls.tunnel_urls, + ) diff --git a/packages/modal-infra/src/web_api.py b/packages/modal-infra/src/web_api.py index 73d76b3acb..16e3677a52 100644 --- a/packages/modal-infra/src/web_api.py +++ b/packages/modal-infra/src/web_api.py @@ -42,6 +42,7 @@ InvalidDockerSettingsError, ModalBackend, ) +from .sandbox.manager import VMAllocationOutcome configure_logging() log = get_logger("web_api") @@ -184,6 +185,13 @@ class RestoreSandboxRequest(_ModalRequestModel): launch_deadline_at_ms: int | None = Field(default=None, gt=0) +class ResolveVMSandboxRequest(_ModalRequestModel): + model_config = ConfigDict(extra="forbid", strict=True) + + session_id: NonEmptyString + sandbox_id: NonEmptyString + + @dataclass class _EndpointExecution: endpoint_name: str @@ -232,6 +240,10 @@ async def _execute_endpoint( execution.http_status = 501 execution.outcome = "error" raise HTTPException(status_code=501, detail="docker_not_available") from e + except VMAllocationOutcome as e: + execution.http_status = 409 + execution.outcome = "error" + raise HTTPException(status_code=execution.http_status, detail=e.detail) from e except Exception as e: execution.http_status = 500 execution.outcome = "error" @@ -286,6 +298,8 @@ def _parse_request[RequestModelT: BaseModel]( }.get(error_type, "user_env_vars has an invalid value") elif field == "timeout_seconds": detail = "timeout_seconds must be a positive integer" + elif error_type == "extra_forbidden": + detail = f"{field} is not allowed" elif len(location) > 1: detail = f"{field} has an invalid value" else: @@ -497,6 +511,48 @@ async def api_create_sandbox( } +@app.function(image=function_image, secrets=[internal_api_secret], timeout=150) +@fastapi_endpoint(method="POST") +async def api_resolve_vm_sandbox( + request: dict, + authorization: str | None = Header(None), + x_trace_id: str | None = Header(None), + x_request_id: str | None = Header(None), + x_session_id: str | None = Header(None), + x_sandbox_id: str | None = Header(None), +) -> dict: + """Authenticated lookup-only VM recovery by session and generation; no create or retire. + + POST body: {"session_id": "...", "sandbox_id": "..."}. No secrets or launch settings. + """ + async with _execute_endpoint( + endpoint_name="api_resolve_vm_sandbox", + authorization=authorization, + trace_id=x_trace_id, + request_id=x_request_id, + session_id=x_session_id, + sandbox_id=x_sandbox_id, + ): + parsed = _parse_request(ResolveVMSandboxRequest, request) + from .sandbox.manager import SandboxManager + + handle = await SandboxManager().resolve_vm_sandbox(parsed.session_id, parsed.sandbox_id) + return { + "success": True, + "data": { + "sandbox_id": handle.sandbox_id, + "modal_object_id": handle.modal_object_id, + "code_server_url": handle.code_server_url, + "code_server_password": handle.code_server_password, + "vnc_url": handle.vnc_url, + "vnc_password": handle.vnc_password, + "ttyd_url": handle.ttyd_url, + "tunnel_urls": handle.tunnel_urls, + "sandbox_backend": handle.sandbox_backend, + }, + } + + @app.function(image=function_image) def deployment_vm_image() -> str | None: """Private SDK-only deployment handshake; preserve capability across selector cutover.""" diff --git a/packages/modal-infra/tests/test_agent_slack_notify_env.py b/packages/modal-infra/tests/test_agent_slack_notify_env.py index abc20cce92..ede0a3c971 100644 --- a/packages/modal-infra/tests/test_agent_slack_notify_env.py +++ b/packages/modal-infra/tests/test_agent_slack_notify_env.py @@ -5,6 +5,7 @@ import pytest from src.sandbox.manager import SandboxConfig, SandboxManager +from src.sandbox.tunnels import SandboxTunnels, TunnelUrls def _patch_create(monkeypatch, captured: dict) -> None: @@ -21,11 +22,11 @@ class FakeSandbox: fake_create = MagicMock() fake_create.aio = fake_create_aio - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", fake_create) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", fake_create) monkeypatch.setattr( - SandboxManager, - "_resolve_and_setup_tunnels", - AsyncMock(return_value=(None, None, None, None)), + SandboxTunnels, + "resolve", + AsyncMock(return_value=TunnelUrls(None, None, None, None)), ) @@ -79,7 +80,7 @@ async def test_env_set_when_enabled(self, monkeypatch): class FakeImage: object_id = "img-123" - monkeypatch.setattr("src.sandbox.manager.modal.Image.from_id", lambda *a, **k: FakeImage()) + monkeypatch.setattr("src.sandbox.launch.modal.Image.from_id", lambda *a, **k: FakeImage()) _patch_create(monkeypatch, captured) manager = SandboxManager() diff --git a/packages/modal-infra/tests/test_code_server.py b/packages/modal-infra/tests/test_code_server.py index f950e2a31b..302d8c7675 100644 --- a/packages/modal-infra/tests/test_code_server.py +++ b/packages/modal-infra/tests/test_code_server.py @@ -4,25 +4,27 @@ import pytest +from src.sandbox.launch import SandboxLauncher from src.sandbox.manager import CODE_SERVER_PORT, SandboxConfig, SandboxManager +from src.sandbox.tunnels import SandboxTunnels, TunnelUrls class TestGenerateCodeServerPassword: - """SandboxManager._generate_code_server_password tests.""" + """SandboxLauncher._generate_code_server_password tests.""" def test_returns_nonempty_password(self): - password = SandboxManager._generate_code_server_password() + password = SandboxLauncher._generate_code_server_password() assert len(password) > 0 def test_generates_unique_passwords(self): passwords = set() for _ in range(20): - passwords.add(SandboxManager._generate_code_server_password()) + passwords.add(SandboxLauncher._generate_code_server_password()) assert len(passwords) == 20 class TestResolveCodeServerTunnel: - """SandboxManager._resolve_tunnels tests for code-server port.""" + """SandboxTunnels._resolve_tunnels tests for code-server port.""" @pytest.mark.asyncio async def test_returns_tunnel_url_on_success(self): @@ -32,7 +34,7 @@ async def test_returns_tunnel_url_on_success(self): sandbox = MagicMock() sandbox.tunnels.return_value = {CODE_SERVER_PORT: tunnel} - resolved = await SandboxManager._resolve_tunnels(sandbox, "sb-123", [CODE_SERVER_PORT]) + resolved = await SandboxTunnels._resolve_tunnels(sandbox, "sb-123", [CODE_SERVER_PORT]) assert resolved.get(CODE_SERVER_PORT) == "https://tunnel.example.com" @pytest.mark.asyncio @@ -40,9 +42,9 @@ async def test_returns_empty_on_exception_after_retries(self): sandbox = MagicMock() sandbox.tunnels.side_effect = Exception("tunnel unavailable") - with patch("src.sandbox.manager.asyncio.sleep", new_callable=AsyncMock): - resolved = await SandboxManager._resolve_tunnels( - sandbox, "sb-123", [CODE_SERVER_PORT], retries=2, backoff=0.0 + with patch("src.sandbox.tunnels.asyncio.sleep", new_callable=AsyncMock): + resolved = await SandboxTunnels._resolve_tunnels( + sandbox, "sb-123", [CODE_SERVER_PORT], retries=2, backoff_seconds=0.0 ) assert resolved == {} assert sandbox.tunnels.call_count == 2 @@ -52,9 +54,9 @@ async def test_returns_empty_when_port_missing_after_retries(self): sandbox = MagicMock() sandbox.tunnels.return_value = {} # no entry for CODE_SERVER_PORT - with patch("src.sandbox.manager.asyncio.sleep", new_callable=AsyncMock): - resolved = await SandboxManager._resolve_tunnels( - sandbox, "sb-123", [CODE_SERVER_PORT], retries=2, backoff=0.0 + with patch("src.sandbox.tunnels.asyncio.sleep", new_callable=AsyncMock): + resolved = await SandboxTunnels._resolve_tunnels( + sandbox, "sb-123", [CODE_SERVER_PORT], retries=2, backoff_seconds=0.0 ) assert resolved == {} @@ -69,9 +71,9 @@ async def test_retries_then_succeeds(self): {CODE_SERVER_PORT: tunnel}, ] - with patch("src.sandbox.manager.asyncio.sleep", new_callable=AsyncMock): - resolved = await SandboxManager._resolve_tunnels( - sandbox, "sb-123", [CODE_SERVER_PORT], retries=3, backoff=0.0 + with patch("src.sandbox.tunnels.asyncio.sleep", new_callable=AsyncMock): + resolved = await SandboxTunnels._resolve_tunnels( + sandbox, "sb-123", [CODE_SERVER_PORT], retries=3, backoff_seconds=0.0 ) assert resolved.get(CODE_SERVER_PORT) == "https://tunnel.example.com" assert sandbox.tunnels.call_count == 2 @@ -96,12 +98,12 @@ class FakeSandbox: fake_create = MagicMock() fake_create.aio = fake_create_aio - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", fake_create) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", fake_create) monkeypatch.setattr( - SandboxManager, - "_resolve_and_setup_tunnels", - AsyncMock(return_value=("https://cs.example.com", None, None, None)), + SandboxTunnels, + "resolve", + AsyncMock(return_value=TunnelUrls("https://cs.example.com", None, None, None)), ) manager = SandboxManager() @@ -140,10 +142,10 @@ class FakeSandbox: fake_create = MagicMock() fake_create.aio = fake_create_aio - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", fake_create) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", fake_create) - tunnel_mock = AsyncMock(return_value=(None, None, None, None)) - monkeypatch.setattr(SandboxManager, "_resolve_and_setup_tunnels", tunnel_mock) + tunnel_mock = AsyncMock(return_value=TunnelUrls(None, None, None, None)) + monkeypatch.setattr(SandboxTunnels, "resolve", tunnel_mock) manager = SandboxManager() config = SandboxConfig( @@ -187,12 +189,12 @@ class FakeSandbox: fake_create = MagicMock() fake_create.aio = fake_create_aio - monkeypatch.setattr("src.sandbox.manager.modal.Image.from_id", fake_from_id) - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", fake_create) + monkeypatch.setattr("src.sandbox.launch.modal.Image.from_id", fake_from_id) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", fake_create) monkeypatch.setattr( - SandboxManager, - "_resolve_and_setup_tunnels", - AsyncMock(return_value=("https://cs-restored.example.com", None, None, None)), + SandboxTunnels, + "resolve", + AsyncMock(return_value=TunnelUrls("https://cs-restored.example.com", None, None, None)), ) manager = SandboxManager() @@ -238,10 +240,10 @@ class FakeSandbox: fake_create = MagicMock() fake_create.aio = fake_create_aio - monkeypatch.setattr("src.sandbox.manager.modal.Image.from_id", fake_from_id) - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", fake_create) - tunnel_mock = AsyncMock(return_value=(None, None, None, None)) - monkeypatch.setattr(SandboxManager, "_resolve_and_setup_tunnels", tunnel_mock) + monkeypatch.setattr("src.sandbox.launch.modal.Image.from_id", fake_from_id) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", fake_create) + tunnel_mock = AsyncMock(return_value=TunnelUrls(None, None, None, None)) + monkeypatch.setattr(SandboxTunnels, "resolve", tunnel_mock) manager = SandboxManager() handle = await manager.restore_from_snapshot( diff --git a/packages/modal-infra/tests/test_docker_launch.py b/packages/modal-infra/tests/test_docker_launch.py index 1b5aacfac2..36d82b37bb 100644 --- a/packages/modal-infra/tests/test_docker_launch.py +++ b/packages/modal-infra/tests/test_docker_launch.py @@ -18,6 +18,7 @@ parse_launch, parse_pending_vm_reference, ) +from src.sandbox.vm_recovery import VMServiceLaunch, parse_vm_service_launch @pytest.mark.parametrize("settings", [None, {}]) @@ -90,6 +91,15 @@ def test_allocation_tags_bind_session_generation_and_backend(): assert re.fullmatch(r"[a-zA-Z0-9._-]{1,63}", value) +def test_vm_launch_metadata_round_trips_at_max_extra_port_count(): + launch = VMServiceLaunch(True, False, True, 9000, 6080, 7680, list(range(60000, 60010))) + tags = launch.tags() + + assert parse_vm_service_launch(tags) == launch + for value in tags.values(): + assert re.fullmatch(r"[a-zA-Z0-9._-]{1,63}", value) + + def test_pending_vm_reference_uses_shared_two_part_wire_format(): reference = 'modal-vm-session:["session-1","sandbox-1"]' assert parse_pending_vm_reference(reference) == ("session-1", "sandbox-1") diff --git a/packages/modal-infra/tests/test_llm_secrets.py b/packages/modal-infra/tests/test_llm_secrets.py index daf8935755..a6448ecc7b 100644 --- a/packages/modal-infra/tests/test_llm_secrets.py +++ b/packages/modal-infra/tests/test_llm_secrets.py @@ -26,7 +26,7 @@ class FakeSandbox: return FakeSandbox() fake_create_aio.aio = fake_create_aio - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", fake_create_aio) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", fake_create_aio) return captured @@ -44,7 +44,7 @@ async def test_restore_attaches_the_deployment_wide_secret( class FakeImage: object_id = "img-llm-secrets" - monkeypatch.setattr("src.sandbox.manager.modal.Image.from_id", lambda *a, **k: FakeImage()) + monkeypatch.setattr("src.sandbox.launch.modal.Image.from_id", lambda *a, **k: FakeImage()) await SandboxManager().restore_from_snapshot( snapshot_image_id="img-abc", diff --git a/packages/modal-infra/tests/test_manager_exports.py b/packages/modal-infra/tests/test_manager_exports.py new file mode 100644 index 0000000000..cddf8c6564 --- /dev/null +++ b/packages/modal-infra/tests/test_manager_exports.py @@ -0,0 +1,45 @@ +"""Compatibility coverage for the manager's pre-refactor constant imports.""" + +import pytest + +from sandbox_runtime import constants +from sandbox_runtime.docker_control import CONTROL_TIMEOUT_SECONDS +from src.app_config import APP_NAME +from src.sandbox import manager +from src.sandbox.launch import ACCESS_PASSWORD_READ_TIMEOUT_SECONDS +from src.sandbox.launch_policy import PENDING_VM_REFERENCE_PREFIX +from src.sandbox.models import DEFAULT_VNC_ENABLED +from src.sandbox.tunnels import MAX_TUNNEL_PORTS + + +@pytest.mark.parametrize( + "name, expected", + [ + ("ACCESS_PASSWORD_READ_TIMEOUT_SECONDS", ACCESS_PASSWORD_READ_TIMEOUT_SECONDS), + ("APP_NAME", APP_NAME), + ("CODE_SERVER_PORT", constants.CODE_SERVER_PORT), + ("CODE_SERVER_PORT_ENV_VAR", constants.CODE_SERVER_PORT_ENV_VAR), + ("DEFAULT_SANDBOX_TIMEOUT_SECONDS", constants.DEFAULT_SANDBOX_TIMEOUT_SECONDS), + ("DEFAULT_VNC_ENABLED", DEFAULT_VNC_ENABLED), + ("DOCKER_ENABLED_ENV_VAR", constants.DOCKER_ENABLED_ENV_VAR), + ("EXPECTED_TUNNEL_PORTS_ENV_VAR", constants.EXPECTED_TUNNEL_PORTS_ENV_VAR), + ("MAX_TUNNEL_PORTS", MAX_TUNNEL_PORTS), + ("NOVNC_PORT", constants.NOVNC_PORT), + ("NOVNC_PORT_ENV_VAR", constants.NOVNC_PORT_ENV_VAR), + ("PENDING_VM_REFERENCE_PREFIX", PENDING_VM_REFERENCE_PREFIX), + ("SANDBOX_TIMEOUT_ENV_VAR", constants.SANDBOX_TIMEOUT_ENV_VAR), + ("SNAPSHOT_FILESYSTEM_TIMEOUT_SECONDS", 300), + ("CONTROL_TIMEOUT_SECONDS", CONTROL_TIMEOUT_SECONDS), + ("TTYD_PROXY_PORT", constants.TTYD_PROXY_PORT), + ("TTYD_PROXY_PORT_ENV_VAR", constants.TTYD_PROXY_PORT_ENV_VAR), + ("TUNNEL_ENV_FILE_PATH", constants.TUNNEL_ENV_FILE_PATH), + ("TUNNEL_ENV_SANDBOX_ID_KEY", constants.TUNNEL_ENV_SANDBOX_ID_KEY), + ("VNC_PASSWORD_ENV_VAR", constants.VNC_PASSWORD_ENV_VAR), + ("VNC_PASSWORD_MAX_BYTES", constants.VNC_PASSWORD_MAX_BYTES), + ("VNC_PORT", constants.VNC_PORT), + ], +) +def test_legacy_manager_constant_exports(name, expected): + """Legacy import names retain their values and are explicitly public.""" + assert getattr(manager, name) == expected + assert name in manager.__all__ diff --git a/packages/modal-infra/tests/test_sandbox_env_vars.py b/packages/modal-infra/tests/test_sandbox_env_vars.py index 131ff0ef1f..be4130a938 100644 --- a/packages/modal-infra/tests/test_sandbox_env_vars.py +++ b/packages/modal-infra/tests/test_sandbox_env_vars.py @@ -8,6 +8,7 @@ VNC_PASSWORD_MAX_BYTES, ) from sandbox_runtime.types import SessionConfig +from src.sandbox.launch import SandboxLauncher from src.sandbox.manager import ( DEFAULT_SANDBOX_TIMEOUT_SECONDS, SandboxConfig, @@ -86,7 +87,7 @@ class FakeSandbox: return FakeSandbox() fake_create_aio.aio = fake_create_aio - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", fake_create_aio) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", fake_create_aio) manager = SandboxManager() config = SandboxConfig( @@ -132,8 +133,8 @@ class FakeSandbox: return FakeSandbox() fake_create_aio.aio = fake_create_aio - monkeypatch.setattr("src.sandbox.manager.modal.Image.from_id", fake_from_id) - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", fake_create_aio) + monkeypatch.setattr("src.sandbox.launch.modal.Image.from_id", fake_from_id) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", fake_create_aio) manager = SandboxManager() await manager.restore_from_snapshot( @@ -179,7 +180,7 @@ async def test_create_preserves_managed_provider_env_isolation( monkeypatch, managed_marker, suppressed_api_key ): captured = {} - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", _fake_sandbox_create(captured)) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", _fake_sandbox_create(captured)) await SandboxManager().create_sandbox( SandboxConfig( @@ -217,7 +218,7 @@ async def test_restore_preserves_managed_provider_env_isolation( def test_generated_vnc_password_respects_protocol_limit(): - assert len(SandboxManager._generate_vnc_password().encode()) == VNC_PASSWORD_MAX_BYTES + assert len(SandboxLauncher._generate_vnc_password().encode()) == VNC_PASSWORD_MAX_BYTES @pytest.mark.asyncio @@ -241,8 +242,8 @@ class FakeSandbox: return FakeSandbox() fake_create_aio.aio = fake_create_aio - monkeypatch.setattr("src.sandbox.manager.modal.Image.from_id", fake_from_id) - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", fake_create_aio) + monkeypatch.setattr("src.sandbox.launch.modal.Image.from_id", fake_from_id) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", fake_create_aio) manager = SandboxManager() await manager.restore_from_snapshot( @@ -281,8 +282,8 @@ class FakeSandbox: return FakeSandbox() fake_create_aio.aio = fake_create_aio - monkeypatch.setattr("src.sandbox.manager.modal.Image.from_id", fake_from_id) - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", fake_create_aio) + monkeypatch.setattr("src.sandbox.launch.modal.Image.from_id", fake_from_id) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", fake_create_aio) manager = SandboxManager() await manager.restore_from_snapshot( @@ -327,8 +328,8 @@ class FakeSandbox: return FakeSandbox() fake_create_aio.aio = fake_create_aio - monkeypatch.setattr("src.sandbox.manager.modal.Image.from_id", fake_from_id) - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", fake_create_aio) + monkeypatch.setattr("src.sandbox.launch.modal.Image.from_id", fake_from_id) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", fake_create_aio) manager = SandboxManager() @@ -369,8 +370,8 @@ def _fake_restore_setup(monkeypatch): class FakeImage: object_id = "img-123" - monkeypatch.setattr("src.sandbox.manager.modal.Image.from_id", lambda *a, **kw: FakeImage()) - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", _fake_sandbox_create(captured)) + monkeypatch.setattr("src.sandbox.launch.modal.Image.from_id", lambda *a, **kw: FakeImage()) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", _fake_sandbox_create(captured)) return captured @@ -466,7 +467,7 @@ class FakeSandbox: async def test_vcs_env_vars_default_github(monkeypatch): """SCM_PROVIDER unset → github.com defaults, no token in env.""" captured = {} - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", _fake_sandbox_create(captured)) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", _fake_sandbox_create(captured)) monkeypatch.delenv("SCM_PROVIDER", raising=False) manager = SandboxManager() @@ -488,7 +489,7 @@ async def test_vcs_env_vars_default_github(monkeypatch): async def test_vcs_env_vars_gitlab(monkeypatch): """SCM_PROVIDER=gitlab → gitlab.com + oauth2, no token in env.""" captured = {} - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", _fake_sandbox_create(captured)) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", _fake_sandbox_create(captured)) monkeypatch.setenv("SCM_PROVIDER", "gitlab") manager = SandboxManager() @@ -508,7 +509,7 @@ async def test_vcs_env_vars_gitlab(monkeypatch): async def test_vcs_env_vars_bitbucket(monkeypatch): """SCM_PROVIDER=bitbucket → bitbucket.org + x-token-auth, no token in env.""" captured = {} - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", _fake_sandbox_create(captured)) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", _fake_sandbox_create(captured)) monkeypatch.setenv("SCM_PROVIDER", "bitbucket") manager = SandboxManager() @@ -532,8 +533,8 @@ async def test_repo_image_boot_omits_fallback_tokens(monkeypatch): class FakeImage: object_id = "repo-img-1" - monkeypatch.setattr("src.sandbox.manager.modal.Image.from_id", lambda *a, **kw: FakeImage()) - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", _fake_sandbox_create(captured)) + monkeypatch.setattr("src.sandbox.launch.modal.Image.from_id", lambda *a, **kw: FakeImage()) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", _fake_sandbox_create(captured)) monkeypatch.delenv("SCM_PROVIDER", raising=False) manager = SandboxManager() @@ -561,8 +562,8 @@ async def test_repo_image_boot_preserves_user_github_cli_token(monkeypatch, toke class FakeImage: object_id = "repo-img-1" - monkeypatch.setattr("src.sandbox.manager.modal.Image.from_id", lambda *a, **kw: FakeImage()) - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", _fake_sandbox_create(captured)) + monkeypatch.setattr("src.sandbox.launch.modal.Image.from_id", lambda *a, **kw: FakeImage()) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", _fake_sandbox_create(captured)) monkeypatch.delenv("SCM_PROVIDER", raising=False) manager = SandboxManager() @@ -591,7 +592,7 @@ async def test_no_repo_sandbox_gets_provider_host_scoping(monkeypatch): fall back to github.com credential-helper behavior. """ captured = {} - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", _fake_sandbox_create(captured)) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", _fake_sandbox_create(captured)) monkeypatch.setenv("SCM_PROVIDER", "gitlab") manager = SandboxManager() @@ -611,8 +612,8 @@ async def test_restore_no_repo_gets_host_scoping_without_tokens(monkeypatch): class FakeImage: object_id = "img-123" - monkeypatch.setattr("src.sandbox.manager.modal.Image.from_id", lambda *a, **kw: FakeImage()) - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", _fake_sandbox_create(captured)) + monkeypatch.setattr("src.sandbox.launch.modal.Image.from_id", lambda *a, **kw: FakeImage()) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", _fake_sandbox_create(captured)) monkeypatch.setenv("SCM_PROVIDER", "bitbucket") manager = SandboxManager() @@ -649,8 +650,8 @@ async def test_restore_preserves_vcs_clone_token_for_legacy_snapshots(monkeypatc class FakeImage: object_id = "img-123" - monkeypatch.setattr("src.sandbox.manager.modal.Image.from_id", lambda *a, **kw: FakeImage()) - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", _fake_sandbox_create(captured)) + monkeypatch.setattr("src.sandbox.launch.modal.Image.from_id", lambda *a, **kw: FakeImage()) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", _fake_sandbox_create(captured)) monkeypatch.setenv("SCM_PROVIDER", "bitbucket") manager = SandboxManager() @@ -685,8 +686,8 @@ async def test_restore_github_includes_gh_cli_aliases(monkeypatch): class FakeImage: object_id = "img-123" - monkeypatch.setattr("src.sandbox.manager.modal.Image.from_id", lambda *a, **kw: FakeImage()) - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", _fake_sandbox_create(captured)) + monkeypatch.setattr("src.sandbox.launch.modal.Image.from_id", lambda *a, **kw: FakeImage()) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", _fake_sandbox_create(captured)) monkeypatch.delenv("SCM_PROVIDER", raising=False) manager = SandboxManager() @@ -720,8 +721,8 @@ async def test_no_repo_restore_omits_clone_token(monkeypatch): class FakeImage: object_id = "img-123" - monkeypatch.setattr("src.sandbox.manager.modal.Image.from_id", lambda *a, **kw: FakeImage()) - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", _fake_sandbox_create(captured)) + monkeypatch.setattr("src.sandbox.launch.modal.Image.from_id", lambda *a, **kw: FakeImage()) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", _fake_sandbox_create(captured)) monkeypatch.delenv("SCM_PROVIDER", raising=False) manager = SandboxManager() diff --git a/packages/modal-infra/tests/test_sandbox_launch.py b/packages/modal-infra/tests/test_sandbox_launch.py index 1fb73b3e32..ec83ce45d8 100644 --- a/packages/modal-infra/tests/test_sandbox_launch.py +++ b/packages/modal-infra/tests/test_sandbox_launch.py @@ -6,6 +6,7 @@ from unittest.mock import AsyncMock, Mock import pytest +from modal.exception import NotFoundError from sandbox_runtime.constants import ( CODE_SERVER_PORT_ENV_VAR, @@ -13,9 +14,12 @@ EXPECTED_TUNNEL_PORTS_ENV_VAR, NOVNC_PORT_ENV_VAR, TTYD_PROXY_PORT_ENV_VAR, + TUNNEL_ENV_FILE_PATH, + TUNNEL_ENV_SANDBOX_ID_KEY, VNC_PASSWORD_ENV_VAR, ) -from sandbox_runtime.types import SessionConfig +from sandbox_runtime.types import SandboxStatus, SessionConfig +from src.sandbox.launch import SandboxLauncher from src.sandbox.launch_policy import ( DockerImageUnavailableError, InvalidDockerSettingsError, @@ -27,13 +31,26 @@ SandboxConfig, SandboxManager, ) +from src.sandbox.tunnels import SandboxTunnels, TunnelUrls +from src.sandbox.vm_recovery import VMAllocationOutcome, VMServiceLaunch def _fake_create(captured: dict): async def create_aio(*args, **kwargs): captured["command"] = args captured["kwargs"] = kwargs - return SimpleNamespace(object_id="modal-object-1", stdout=None) + return SimpleNamespace( + object_id="modal-object-1", + tunnels=Mock( + return_value={ + 9000: SimpleNamespace(url="https://code.example"), + 9001: SimpleNamespace(url="https://vnc.example"), + 9002: SimpleNamespace(url="https://terminal.example"), + 3000: SimpleNamespace(url="https://app.example"), + } + ), + filesystem=SimpleNamespace(write_text=SimpleNamespace(aio=AsyncMock())), + ) create_aio.aio = create_aio return create_aio @@ -50,27 +67,14 @@ async def test_launch_matrix_preserves_common_and_source_specific_behavior( "repo-image-1": object(), "snapshot-image-1": object(), } - monkeypatch.setattr("src.sandbox.manager.base_image", base_image) - monkeypatch.setattr("src.sandbox.manager.modal.Image.from_id", images.__getitem__) - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", _fake_create(captured)) + monkeypatch.setattr("src.sandbox.launch.base_image", base_image) + monkeypatch.setattr("src.sandbox.launch.modal.Image.from_id", images.__getitem__) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", _fake_create(captured)) monkeypatch.delenv("SCM_PROVIDER", raising=False) - resolve_tunnels = AsyncMock( - return_value=( - "https://code.example", - "https://vnc.example", - "https://terminal.example", - {3000: "https://app.example"}, - ) - ) monkeypatch.setattr( - SandboxManager, - "_resolve_and_setup_tunnels", - resolve_tunnels, + SandboxLauncher, "_generate_code_server_password", staticmethod(lambda: "code-password") ) - monkeypatch.setattr( - SandboxManager, "_generate_code_server_password", staticmethod(lambda: "code-password") - ) - monkeypatch.setattr(SandboxManager, "_generate_vnc_password", staticmethod(lambda: "vnc-pass")) + monkeypatch.setattr(SandboxLauncher, "_generate_vnc_password", staticmethod(lambda: "vnc-pass")) manager = SandboxManager() settings = { @@ -189,23 +193,17 @@ async def test_launch_matrix_preserves_common_and_source_specific_behavior( assert handle.vnc_password == "vnc-pass" assert handle.ttyd_url == "https://terminal.example" assert handle.tunnel_urls == {3000: "https://app.example"} - resolve_tunnels.assert_awaited_once_with( - handle.modal_sandbox, - "sandbox-1", - True, - True, - True, - [3000], - 9000, - 9001, - 9002, + handle.modal_sandbox.tunnels.assert_called_once_with() + handle.modal_sandbox.filesystem.write_text.aio.assert_awaited_once_with( + f"{TUNNEL_ENV_SANDBOX_ID_KEY}=sandbox-1\nTUNNEL_3000=https://app.example\n", + TUNNEL_ENV_FILE_PATH, ) @pytest.mark.asyncio async def test_repository_image_create_validates_repo_before_image_lookup(monkeypatch): from_id = Mock(side_effect=AssertionError("image lookup should not run")) - monkeypatch.setattr("src.sandbox.manager.modal.Image.from_id", from_id) + monkeypatch.setattr("src.sandbox.launch.modal.Image.from_id", from_id) with pytest.raises(ValueError, match="repo_owner and repo_name must be provided together"): await SandboxManager().create_sandbox( @@ -216,25 +214,147 @@ async def test_repository_image_create_validates_repo_before_image_lookup(monkey @pytest.mark.asyncio -async def test_repository_image_not_found_is_reported_explicitly(monkeypatch, fake_llm_secret): - from modal.exception import NotFoundError +@pytest.mark.parametrize("image_source", ["repository", "snapshot"]) +@pytest.mark.parametrize("failure_stage", ["lookup", "create"]) +@pytest.mark.parametrize("missing", [False, True]) +async def test_launch_preserves_image_error_classification( + monkeypatch, image_source, failure_stage, missing +): + error = NotFoundError("missing image") if missing else RuntimeError("transient failure") + from_id = Mock( + return_value=object(), + side_effect=error if failure_stage == "lookup" else None, + ) + create = AsyncMock(side_effect=error) + monkeypatch.setattr("src.sandbox.launch.modal.Image.from_id", from_id) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", SimpleNamespace(aio=create)) + expected_error = ( + RepositoryImageUnavailableError if image_source == "repository" and missing else type(error) + ) + + with pytest.raises(expected_error) as raised: + if image_source == "snapshot": + await SandboxManager().restore_from_snapshot( + snapshot_image_id="image-1", + session_config={"repo_owner": "acme", "repo_name": "repo"}, + ) + else: + await SandboxManager().create_sandbox( + SandboxConfig(repo_owner="acme", repo_name="repo", repo_image_id="image-1") + ) - monkeypatch.setattr("src.sandbox.manager.modal.Image.from_id", lambda _image_id: object()) + if expected_error is RepositoryImageUnavailableError: + assert raised.value.__cause__ is error + else: + assert raised.value is error + from_id.assert_called_once_with("image-1") + if failure_stage == "lookup": + create.assert_not_awaited() + else: + # A spawn error must not silently fall back to a different image or retry. + create.assert_awaited_once() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("missing", [False, True]) +async def test_base_image_spawn_errors_propagate_without_retry(monkeypatch, missing): + error = NotFoundError("missing image") if missing else RuntimeError("transient failure") + create = AsyncMock(side_effect=error) + from_id = Mock() + monkeypatch.setattr("src.sandbox.launch.modal.Image.from_id", from_id) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", SimpleNamespace(aio=create)) + + with pytest.raises(type(error)) as raised: + await SandboxManager().create_sandbox(SandboxConfig(repo_owner=None, repo_name=None)) + + assert raised.value is error + create.assert_awaited_once() + from_id.assert_not_called() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("image_source", ["base", "repository", "snapshot"]) +@pytest.mark.parametrize("failure", ["partial", "unavailable", "write"]) +async def test_launch_returns_handle_despite_tunnel_failures(monkeypatch, image_source, failure): + write_text = AsyncMock(side_effect=OSError("write failed") if failure == "write" else None) + sandbox = SimpleNamespace( + object_id="modal-object-1", + tunnels=Mock( + side_effect=( + [RuntimeError("unavailable")] * 3 + if failure == "unavailable" + else [ + {9000: SimpleNamespace(url="https://code.example")}, + RuntimeError("not ready"), + {3000: SimpleNamespace(url="https://app.example")}, + ] + ) + ), + filesystem=SimpleNamespace(write_text=SimpleNamespace(aio=write_text)), + ) + create = AsyncMock(return_value=sandbox) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", SimpleNamespace(aio=create)) + monkeypatch.setattr("src.sandbox.launch.modal.Image.from_id", lambda _: object()) + sleep = AsyncMock() + monkeypatch.setattr("src.sandbox.tunnels.asyncio.sleep", sleep) + common = { + "sandbox_id": "sandbox-partial", + "code_server_enabled": True, + "settings": {"codeServerPort": 9000, "tunnelPorts": [3000, 3001]}, + } + manager = SandboxManager() + + if image_source == "snapshot": + handle = await manager.restore_from_snapshot( + snapshot_image_id="image-1", + session_config={"repo_owner": "acme", "repo_name": "repo"}, + **common, + ) + else: + handle = await manager.create_sandbox( + SandboxConfig( + repo_owner="acme", + repo_name="repo", + repo_image_id="image-1" if image_source == "repository" else None, + **common, + ) + ) + + assert handle.status is SandboxStatus.WARMING + assert handle.modal_sandbox is sandbox + assert handle.modal_object_id == "modal-object-1" + assert handle.code_server_password == create.call_args.kwargs["env"]["CODE_SERVER_PASSWORD"] + assert create.call_args.kwargs["encrypted_ports"] == [9000, 3000, 3001] + assert sandbox.tunnels.call_count == 3 + assert [call.args for call in sleep.await_args_list] == [(1.0,), (2.0,)] + create.assert_awaited_once() + if failure == "unavailable": + assert handle.code_server_url is None + assert handle.tunnel_urls is None + write_text.assert_not_awaited() + else: + assert handle.code_server_url == "https://code.example" + assert handle.tunnel_urls == {3000: "https://app.example"} + write_text.assert_awaited_once_with( + f"{TUNNEL_ENV_SANDBOX_ID_KEY}=sandbox-partial\nTUNNEL_3000=https://app.example\n", + TUNNEL_ENV_FILE_PATH, + ) + + +@pytest.mark.asyncio +async def test_repository_image_not_found_is_reported_explicitly(monkeypatch, fake_llm_secret): + monkeypatch.setattr("src.sandbox.launch.modal.Image.from_id", lambda _image_id: object()) async def create_aio(*_args, **_kwargs): fake_llm_secret[0].hydrate.aio.assert_awaited_once_with() raise NotFoundError("image not found") create = SimpleNamespace(aio=AsyncMock(side_effect=create_aio)) - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", create) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", create) with pytest.raises(RepositoryImageUnavailableError) as exc_info: await SandboxManager().create_sandbox( - SandboxConfig( - repo_owner="acme", - repo_name="repo", - repo_image_id="repo-image-missing", - ) + SandboxConfig(repo_owner="acme", repo_name="repo", repo_image_id="image-1") ) assert isinstance(exc_info.value.__cause__, NotFoundError) @@ -243,17 +363,15 @@ async def create_aio(*_args, **_kwargs): @pytest.mark.asyncio async def test_missing_secret_does_not_mark_repository_image_unavailable(monkeypatch): - from modal.exception import NotFoundError - create = SimpleNamespace(aio=AsyncMock()) - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", create) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", create) def missing_secret(_name, **_kwargs): secret = Mock() secret.hydrate.aio = AsyncMock(side_effect=NotFoundError("secret not found")) return secret - monkeypatch.setattr("src.sandbox.manager.modal.Secret.from_name", missing_secret) + monkeypatch.setattr("src.sandbox.launch.modal.Secret.from_name", missing_secret) with pytest.raises(NotFoundError, match="secret not found"): await SandboxManager().create_sandbox( @@ -265,10 +383,8 @@ def missing_secret(_name, **_kwargs): @pytest.mark.asyncio async def test_base_image_not_found_is_not_classified_as_repository_image(monkeypatch): - from modal.exception import NotFoundError - create = SimpleNamespace(aio=AsyncMock(side_effect=NotFoundError("base image not found"))) - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", create) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", create) with pytest.raises(NotFoundError, match="base image not found"): await SandboxManager().create_sandbox(SandboxConfig(repo_owner="acme", repo_name="repo")) @@ -282,13 +398,12 @@ async def test_base_image_not_found_is_not_classified_as_repository_image(monkey def _docker_manager(monkeypatch) -> tuple[SandboxManager, dict, object]: captured: dict = {} docker_image = object() - monkeypatch.setattr("src.sandbox.manager.base_image", object()) + monkeypatch.setattr("src.sandbox.launch.base_image", object()) monkeypatch.setattr("src.images.base.docker_image", docker_image) - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", _fake_create(captured)) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", _fake_create(captured)) monkeypatch.setattr( - SandboxManager, - "_resolve_and_setup_tunnels", - AsyncMock(return_value=(None, None, None, {})), + "src.sandbox.tunnels.SandboxTunnels.resolve", + AsyncMock(return_value=TunnelUrls()), ) return SandboxManager(), captured, docker_image @@ -311,8 +426,6 @@ def _docker_config(**overrides) -> SandboxConfig: def _not_found(*_args, **_kwargs): - from modal.exception import NotFoundError - raise NotFoundError("no sandbox") @@ -321,9 +434,9 @@ def _not_found(*_args, **_kwargs): async def test_docker_launch_selects_vm_runtime_and_named_allocation(monkeypatch, image_source): manager, captured, docker_image = _docker_manager(monkeypatch) artifact = object() - monkeypatch.setattr("src.sandbox.manager.modal.Image.from_id", lambda _id: artifact) + monkeypatch.setattr("src.sandbox.launch.modal.Image.from_id", lambda _id: artifact) monkeypatch.setattr( - "src.sandbox.manager.modal.Sandbox.from_name", + "src.sandbox.launch.modal.Sandbox.from_name", SimpleNamespace(aio=AsyncMock(side_effect=_not_found)), ) @@ -349,19 +462,78 @@ async def test_docker_launch_selects_vm_runtime_and_named_allocation(monkeypatch assert kwargs["cpu"] == (2.0, 2.0) assert kwargs["memory"] == 4096 assert kwargs["name"] == docker_allocation_name("session-1") - assert kwargs["tags"] == docker_allocation_tags("session-1", "sandbox-acme-repo-1700000000000") - # The trusted signal wins over any user-supplied value. + assert kwargs["tags"] == { + **docker_allocation_tags("session-1", "sandbox-acme-repo-1700000000000"), + **VMServiceLaunch(False, False, False, 8080, 6080, 7680, []).tags(), + } assert kwargs["env"][DOCKER_ENABLED_ENV_VAR] == "true" assert handle.sandbox_backend == "modal-vm" +@pytest.mark.asyncio +async def test_docker_launch_does_not_allow_user_env_to_spoof_resolved_access(monkeypatch): + manager, captured, _ = _docker_manager(monkeypatch) + monkeypatch.setattr( + "src.sandbox.launch.modal.Sandbox.from_name", + SimpleNamespace(aio=AsyncMock(side_effect=_not_found)), + ) + await manager.create_sandbox( + _docker_config( + user_env_vars={ + "CODE_SERVER_PASSWORD": "spoofed", + VNC_PASSWORD_ENV_VAR: "spoofed", + CODE_SERVER_PORT_ENV_VAR: "9000", + EXPECTED_TUNNEL_PORTS_ENV_VAR: "3000", + } + ) + ) + + for key in ( + "CODE_SERVER_PASSWORD", + VNC_PASSWORD_ENV_VAR, + CODE_SERVER_PORT_ENV_VAR, + EXPECTED_TUNNEL_PORTS_ENV_VAR, + ): + assert key not in captured["kwargs"]["env"] + assert captured["kwargs"]["tags"]["openinspect_vm_launch"] == "1-000-8080-6080-7680" + assert captured["kwargs"]["tags"]["openinspect_vm_ports"] == "none" + + +@pytest.mark.asyncio +async def test_docker_launch_tags_record_effective_enabled_services_and_ports(monkeypatch): + manager, captured, _ = _docker_manager(monkeypatch) + monkeypatch.setattr( + "src.sandbox.launch.modal.Sandbox.from_name", + SimpleNamespace(aio=AsyncMock(side_effect=_not_found)), + ) + + await manager.create_sandbox( + _docker_config( + code_server_enabled=True, + vnc_enabled=True, + settings={ + **DOCKER_SETTINGS, + "terminalEnabled": True, + "codeServerPort": 9000, + "vncPort": 9001, + "terminalPort": 9002, + "tunnelPorts": [3000, 3001], + }, + ) + ) + + assert captured["kwargs"]["tags"]["openinspect_vm_launch"] == "1-111-9000-9001-9002" + assert captured["kwargs"]["tags"]["openinspect_vm_ports"] == "3000-3001" + assert captured["kwargs"]["encrypted_ports"] == [9000, 9001, 9002, 3000, 3001] + + @pytest.mark.asyncio @pytest.mark.parametrize("image_source", ["base", "snapshot"]) async def test_expired_vm_launch_cannot_materialize_after_lookup(monkeypatch, image_source): manager, captured, _ = _docker_manager(monkeypatch) - monkeypatch.setattr("src.sandbox.manager.modal.Image.from_id", lambda _id: object()) + monkeypatch.setattr("src.sandbox.launch.modal.Image.from_id", lambda _id: object()) monkeypatch.setattr( - "src.sandbox.manager.modal.Sandbox.from_name", + "src.sandbox.launch.modal.Sandbox.from_name", SimpleNamespace(aio=AsyncMock(side_effect=_not_found)), ) if image_source == "base": @@ -375,8 +547,9 @@ async def test_expired_vm_launch_cannot_materialize_after_lookup(monkeypatch, im sandbox_backend="modal-vm", launch_deadline_at_ms=1, ) - with pytest.raises(RuntimeError, match="launch deadline"): + with pytest.raises(VMAllocationOutcome) as exc: await launch + assert exc.value.detail == "window_closed" assert "kwargs" not in captured @@ -409,7 +582,7 @@ async def test_docker_launch_adopts_an_existing_owned_allocation(monkeypatch): existing.get_tags.aio = existing.get_tags from_name = AsyncMock(return_value=existing) monkeypatch.setattr( - "src.sandbox.manager.modal.Sandbox.from_name", SimpleNamespace(aio=from_name) + "src.sandbox.launch.modal.Sandbox.from_name", SimpleNamespace(aio=from_name) ) handle = await manager.create_sandbox(_docker_config()) @@ -425,19 +598,19 @@ async def test_docker_launch_adopts_an_existing_owned_allocation(monkeypatch): async def test_docker_retry_returns_the_original_access_credentials( monkeypatch, create_race, image_source ): - from modal.exception import AlreadyExistsError, NotFoundError + from modal.exception import AlreadyExistsError manager, captured, _ = _docker_manager(monkeypatch) - monkeypatch.setattr("src.sandbox.manager.modal.Image.from_id", lambda _id: object()) + monkeypatch.setattr("src.sandbox.launch.modal.Image.from_id", lambda _id: object()) monkeypatch.setattr( - SandboxManager, "_generate_code_server_password", Mock(side_effect=["original", "new"]) + SandboxLauncher, "_generate_code_server_password", Mock(side_effect=["original", "new"]) ) monkeypatch.setattr( - SandboxManager, "_generate_vnc_password", Mock(side_effect=["old-vnc", "new-vnc"]) + SandboxLauncher, "_generate_vnc_password", Mock(side_effect=["old-vnc", "new-vnc"]) ) from_name = AsyncMock(side_effect=NotFoundError("not created")) monkeypatch.setattr( - "src.sandbox.manager.modal.Sandbox.from_name", SimpleNamespace(aio=from_name) + "src.sandbox.launch.modal.Sandbox.from_name", SimpleNamespace(aio=from_name) ) async def launch(): @@ -470,7 +643,7 @@ async def launch(): ) from_name.side_effect = [NotFoundError("racing"), existing] if create_race else [existing] create = AsyncMock(side_effect=AlreadyExistsError("already created")) - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", SimpleNamespace(aio=create)) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", SimpleNamespace(aio=create)) adopted = await launch() @@ -503,7 +676,7 @@ async def test_docker_adoption_fails_if_original_credentials_cannot_be_recovered exec=SimpleNamespace(aio=AsyncMock(return_value=process)), ) monkeypatch.setattr( - "src.sandbox.manager.modal.Sandbox.from_name", + "src.sandbox.launch.modal.Sandbox.from_name", SimpleNamespace(aio=AsyncMock(return_value=existing)), ) @@ -511,7 +684,7 @@ async def test_docker_adoption_fails_if_original_credentials_cannot_be_recovered await manager.create_sandbox(_docker_config(code_server_enabled=True)) assert "kwargs" not in captured - manager._resolve_and_setup_tunnels.assert_not_awaited() + SandboxTunnels.resolve.assert_not_awaited() @pytest.mark.asyncio @@ -523,21 +696,25 @@ async def test_docker_launch_refuses_a_same_named_allocation_it_does_not_own(mon ) foreign.get_tags.aio = foreign.get_tags monkeypatch.setattr( - "src.sandbox.manager.modal.Sandbox.from_name", + "src.sandbox.launch.modal.Sandbox.from_name", SimpleNamespace(aio=AsyncMock(return_value=foreign)), ) - with pytest.raises(RuntimeError, match="ownership mismatch") as exc: + with pytest.raises(VMAllocationOutcome, match="ownership mismatch") as exc: await manager.create_sandbox(_docker_config()) - assert type(exc.value) is RuntimeError + assert exc.value.detail == "other_generation" assert "kwargs" not in captured @pytest.mark.asyncio async def test_docker_launch_retires_the_prior_generation_only_when_owned(monkeypatch): manager, captured, _ = _docker_manager(monkeypatch) - prior_tags = docker_allocation_tags("session-1", "sandbox-acme-repo-1699999999999") + prior_tags = { + **docker_allocation_tags("session-1", "sandbox-acme-repo-1699999999999"), + "openinspect_vm_launch": "1-000-8080-6080-7680", + "openinspect_vm_ports": "none", + } prior = SimpleNamespace( object_id="modal-prior", get_tags=AsyncMock(return_value=prior_tags), @@ -553,7 +730,7 @@ async def from_name(_app, name): _not_found() monkeypatch.setattr( - "src.sandbox.manager.modal.Sandbox.from_name", SimpleNamespace(aio=from_name) + "src.sandbox.launch.modal.Sandbox.from_name", SimpleNamespace(aio=from_name) ) await manager.create_sandbox( @@ -563,11 +740,10 @@ async def from_name(_app, name): prior.terminate.assert_awaited_once_with(wait=True) assert captured["kwargs"]["name"] == docker_allocation_name("session-1") - # A prior allocation with foreign tags is left alone. prior.terminate.reset_mock() prior.get_tags = AsyncMock(return_value={"openinspect_kind": "other"}) prior.get_tags.aio = prior.get_tags - with pytest.raises(RuntimeError, match="ownership mismatch"): + with pytest.raises(VMAllocationOutcome, match="ownership mismatch"): await manager.create_sandbox( _docker_config(retire_sandbox_id="sandbox-acme-repo-1699999999999") ) @@ -576,9 +752,10 @@ async def from_name(_app, name): @pytest.mark.asyncio async def test_late_predecessor_cannot_materialize_beside_successor(monkeypatch): - from modal.exception import AlreadyExistsError, NotFoundError + from modal.exception import AlreadyExistsError - manager, _, _ = _docker_manager(monkeypatch) + _, _, _ = _docker_manager(monkeypatch) + launcher = SandboxLauncher() predecessor_name = docker_allocation_name("session-1") predecessor = SimpleNamespace( object_id="late-predecessor", @@ -589,22 +766,22 @@ async def test_late_predecessor_cannot_materialize_beside_successor(monkeypatch) lookup = AsyncMock( side_effect=[NotFoundError("still creating"), NotFoundError("still creating"), predecessor] ) - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.from_name", SimpleNamespace(aio=lookup)) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.from_name", SimpleNamespace(aio=lookup)) async def create(kwargs, *, repository_image): - # Provider-side naming wins the race after both client lookups missed it. if kwargs["name"] == predecessor_name: raise AlreadyExistsError("predecessor won the name") return SimpleNamespace(object_id="duplicate-successor") - monkeypatch.setattr("src.sandbox.manager._create_sandbox", create) - with pytest.raises(RuntimeError, match="ownership mismatch"): - await manager._launch_docker_sandbox( + monkeypatch.setattr("src.sandbox.launch._create_sandbox", create) + with pytest.raises(VMAllocationOutcome, match="ownership mismatch"): + await launcher._launch_docker_sandbox( session_id="session-1", sandbox_id="successor", retire_sandbox_id="prior", create_kwargs={}, repository_image=False, + service_launch=VMServiceLaunch(False, False, False, 8080, 6080, 7680, []), ) @@ -631,10 +808,8 @@ async def terminate(*, wait=False): ), terminate=SimpleNamespace(aio=terminate), ) - from modal.exception import NotFoundError - monkeypatch.setattr( - "src.sandbox.manager.modal.Sandbox.from_name", + "src.sandbox.launch.modal.Sandbox.from_name", SimpleNamespace(aio=AsyncMock(side_effect=[prior, NotFoundError("no successor")])), ) launch = asyncio.create_task( @@ -656,3 +831,50 @@ async def terminate(*, wait=False): finally: launch.cancel() await asyncio.gather(launch, return_exceptions=True) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("restore", [False, True], ids=["create", "restore"]) +@pytest.mark.parametrize( + "ports, expected", + [ + ([True, False], []), + ([True, False, 0, -1, 65536, "3000", 3.5, None, 1, 3000, 65535], [1, 3000, 65535]), + ([True] * 10 + [3000], [3000]), + ], + ids=["booleans-only", "mixed-with-boundary-ports", "booleans-do-not-consume-limit"], +) +async def test_launch_rejects_boolean_tunnel_ports(monkeypatch, restore, ports, expected): + """Invalid extras never reach Modal or the runtime's expected-port list.""" + urls = {port: f"https://port-{port}.example" for port in expected} + sandbox = SimpleNamespace( + object_id="modal-ports", + tunnels=Mock(return_value={port: SimpleNamespace(url=url) for port, url in urls.items()}), + filesystem=SimpleNamespace(write_text=SimpleNamespace(aio=AsyncMock())), + ) + create = AsyncMock(return_value=sandbox) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", SimpleNamespace(aio=create)) + monkeypatch.setattr("src.sandbox.launch.modal.Image.from_id", lambda _: object()) + manager = SandboxManager() + settings = {"tunnelPorts": ports} + + if restore: + handle = await manager.restore_from_snapshot( + snapshot_image_id="image-1", + session_config={"repo_owner": "acme", "repo_name": "repo"}, + settings=settings, + ) + else: + handle = await manager.create_sandbox( + SandboxConfig(repo_owner="acme", repo_name="repo", settings=settings) + ) + + kwargs = create.call_args.kwargs + assert kwargs.get("encrypted_ports", []) == expected + assert all(type(port) is int for port in kwargs.get("encrypted_ports", [])) + assert kwargs["env"].get(EXPECTED_TUNNEL_PORTS_ENV_VAR) == ( + ",".join(str(port) for port in expected) if expected else None + ) + assert handle.tunnel_urls == (urls or None) + if not expected: + sandbox.tunnels.assert_not_called() diff --git a/packages/modal-infra/tests/test_sandbox_resources.py b/packages/modal-infra/tests/test_sandbox_resources.py index cb8d7a1b90..2505511766 100644 --- a/packages/modal-infra/tests/test_sandbox_resources.py +++ b/packages/modal-infra/tests/test_sandbox_resources.py @@ -6,6 +6,7 @@ from src.sandbox.launch_policy import launch_kwargs, parse_launch from src.sandbox.manager import SandboxConfig, SandboxManager +from src.sandbox.tunnels import SandboxTunnels, TunnelUrls class TestResourceKwargs: @@ -47,11 +48,11 @@ class TestCreateSandboxResources: @pytest.mark.asyncio async def test_create_sandbox_passes_cpu_and_memory(self, monkeypatch): captured: dict = {} - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", _fake_create(captured)) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", _fake_create(captured)) monkeypatch.setattr( - SandboxManager, - "_resolve_and_setup_tunnels", - AsyncMock(return_value=(None, None, None, None)), + SandboxTunnels, + "resolve", + AsyncMock(return_value=TunnelUrls(None, None, None, None)), ) manager = SandboxManager() @@ -74,13 +75,13 @@ class FakeImage: object_id = "img-1" monkeypatch.setattr( - "src.sandbox.manager.modal.Image.from_id", lambda *_a, **_kw: FakeImage() + "src.sandbox.launch.modal.Image.from_id", lambda *_a, **_kw: FakeImage() ) - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", _fake_create(captured)) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", _fake_create(captured)) monkeypatch.setattr( - SandboxManager, - "_resolve_and_setup_tunnels", - AsyncMock(return_value=(None, None, None, None)), + SandboxTunnels, + "resolve", + AsyncMock(return_value=TunnelUrls(None, None, None, None)), ) manager = SandboxManager() diff --git a/packages/modal-infra/tests/test_snapshot_timeout.py b/packages/modal-infra/tests/test_snapshot_timeout.py index a4e4eb8ff6..a57d3fa0a5 100644 --- a/packages/modal-infra/tests/test_snapshot_timeout.py +++ b/packages/modal-infra/tests/test_snapshot_timeout.py @@ -21,7 +21,13 @@ async def test_pending_vm_reference_recovers_owned_allocation(monkeypatch): sandbox = SimpleNamespace( object_id="sb-owned", - get_tags=_async_method(docker_allocation_tags("session", "generation")), + get_tags=_async_method( + { + **docker_allocation_tags("session", "generation"), + "openinspect_vm_launch": "1-000-8080-6080-7680", + "openinspect_vm_ports": "none", + } + ), ) lookup = _async_method(sandbox) monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.from_name", lookup) @@ -34,7 +40,13 @@ async def test_pending_vm_reference_recovers_owned_allocation(monkeypatch): async def test_pending_vm_reference_stops_only_its_owned_allocation(monkeypatch): sandbox = SimpleNamespace( object_id="sb-owned", - get_tags=_async_method(docker_allocation_tags("session", "generation")), + get_tags=_async_method( + { + **docker_allocation_tags("session", "generation"), + "openinspect_vm_launch": "1-000-8080-6080-7680", + "openinspect_vm_ports": "none", + } + ), terminate=_async_method(), ) from_name = _async_method(sandbox) diff --git a/packages/modal-infra/tests/test_ttyd.py b/packages/modal-infra/tests/test_ttyd.py index 015cb456e5..3ddd6ae978 100644 --- a/packages/modal-infra/tests/test_ttyd.py +++ b/packages/modal-infra/tests/test_ttyd.py @@ -4,74 +4,38 @@ import pytest -from sandbox_runtime.constants import NOVNC_PORT, TTYD_PORT +from sandbox_runtime.constants import ( + EXPECTED_TUNNEL_PORTS_ENV_VAR, + NOVNC_PORT, + TTYD_PORT, +) from src.sandbox.manager import ( CODE_SERVER_PORT, TTYD_PROXY_PORT, SandboxConfig, SandboxManager, ) +from src.sandbox.tunnels import SandboxTunnels, TunnelUrls -class TestCollectExposedPortsTerminal: - """_collect_exposed_ports with terminal_enabled flag.""" +@pytest.mark.parametrize("code_server", [False, True]) +@pytest.mark.parametrize("terminal", [False, True]) +def test_terminal_port_ownership(code_server, terminal): + tunnels = SandboxTunnels( + code_server_enabled=code_server, + settings={"terminalEnabled": terminal}, + ) + assert (TTYD_PROXY_PORT in tunnels.exposed_ports) is terminal + assert (CODE_SERVER_PORT in tunnels.exposed_ports) is code_server + assert TTYD_PORT not in tunnels.exposed_ports - def test_terminal_enabled_includes_proxy_port(self): - exposed, _extra = SandboxManager._collect_exposed_ports( - code_server_enabled=False, - vnc_enabled=False, - terminal_enabled=True, - settings=None, - code_server_port=CODE_SERVER_PORT, - novnc_port=NOVNC_PORT, - ttyd_proxy_port=TTYD_PROXY_PORT, - ) - assert TTYD_PROXY_PORT in exposed - # ttyd raw port should NOT be exposed (only the proxy port) - assert TTYD_PORT not in exposed - def test_terminal_disabled_excludes_proxy_port(self): - exposed, _extra = SandboxManager._collect_exposed_ports( - code_server_enabled=False, - vnc_enabled=False, - terminal_enabled=False, - settings=None, - code_server_port=CODE_SERVER_PORT, - novnc_port=NOVNC_PORT, - ttyd_proxy_port=TTYD_PROXY_PORT, - ) - assert TTYD_PROXY_PORT not in exposed - - def test_terminal_and_code_server_both_enabled(self): - exposed, _extra = SandboxManager._collect_exposed_ports( - code_server_enabled=True, - vnc_enabled=False, - terminal_enabled=True, - settings=None, - code_server_port=CODE_SERVER_PORT, - novnc_port=NOVNC_PORT, - ttyd_proxy_port=TTYD_PROXY_PORT, - ) - assert CODE_SERVER_PORT in exposed - assert TTYD_PROXY_PORT in exposed - - def test_terminal_port_deduped_from_tunnel_ports(self): - """If user explicitly lists TTYD_PROXY_PORT in tunnelPorts, it should not duplicate.""" - settings = {"tunnelPorts": [TTYD_PROXY_PORT, 3000]} - exposed, extra = SandboxManager._collect_exposed_ports( - code_server_enabled=False, - vnc_enabled=False, - terminal_enabled=True, - settings=settings, - code_server_port=CODE_SERVER_PORT, - novnc_port=NOVNC_PORT, - ttyd_proxy_port=TTYD_PROXY_PORT, - ) - assert exposed.count(TTYD_PROXY_PORT) == 1 - assert 3000 in exposed - # TTYD_PROXY_PORT should not be in extra (reserved) - assert TTYD_PROXY_PORT not in extra - assert 3000 in extra +def test_terminal_port_is_not_duplicated_by_extra_tunnel(): + tunnels = SandboxTunnels( + settings={"terminalEnabled": True, "tunnelPorts": [TTYD_PROXY_PORT, 3000]} + ) + assert tunnels.exposed_ports == [TTYD_PROXY_PORT, 3000] + assert tunnels.environment[EXPECTED_TUNNEL_PORTS_ENV_VAR] == "3000" class TestResolveTunnelsTerminal: @@ -85,17 +49,17 @@ async def test_returns_ttyd_url_when_terminal_enabled(self): sandbox = MagicMock() sandbox.tunnels.return_value = {TTYD_PROXY_PORT: tunnel} - cs_url, vnc_url, ttyd_url, extra = await SandboxManager._resolve_and_setup_tunnels( - sandbox, - "sb-123", + cs_url, vnc_url, ttyd_url, extra = await SandboxTunnels( code_server_enabled=False, vnc_enabled=False, - terminal_enabled=True, - extra_ports=[], - code_server_port=CODE_SERVER_PORT, - novnc_port=NOVNC_PORT, - ttyd_proxy_port=TTYD_PROXY_PORT, - ) + settings={ + "terminalEnabled": True, + "tunnelPorts": [], + "codeServerPort": CODE_SERVER_PORT, + "vncPort": NOVNC_PORT, + "terminalPort": TTYD_PROXY_PORT, + }, + ).resolve(sandbox, "sb-123") assert cs_url is None assert vnc_url is None assert ttyd_url == "https://ttyd.example.com" @@ -104,17 +68,17 @@ async def test_returns_ttyd_url_when_terminal_enabled(self): @pytest.mark.asyncio async def test_returns_none_when_terminal_disabled(self): sandbox = MagicMock() - cs_url, vnc_url, ttyd_url, extra = await SandboxManager._resolve_and_setup_tunnels( - sandbox, - "sb-123", + cs_url, vnc_url, ttyd_url, extra = await SandboxTunnels( code_server_enabled=False, vnc_enabled=False, - terminal_enabled=False, - extra_ports=[], - code_server_port=CODE_SERVER_PORT, - novnc_port=NOVNC_PORT, - ttyd_proxy_port=TTYD_PROXY_PORT, - ) + settings={ + "terminalEnabled": False, + "tunnelPorts": [], + "codeServerPort": CODE_SERVER_PORT, + "vncPort": NOVNC_PORT, + "terminalPort": TTYD_PROXY_PORT, + }, + ).resolve(sandbox, "sb-123") assert cs_url is None assert vnc_url is None assert ttyd_url is None @@ -133,17 +97,17 @@ async def test_both_code_server_and_terminal(self): TTYD_PROXY_PORT: ttyd_tunnel, } - cs_url, vnc_url, ttyd_url, extra = await SandboxManager._resolve_and_setup_tunnels( - sandbox, - "sb-123", + cs_url, vnc_url, ttyd_url, extra = await SandboxTunnels( code_server_enabled=True, vnc_enabled=False, - terminal_enabled=True, - extra_ports=[], - code_server_port=CODE_SERVER_PORT, - novnc_port=NOVNC_PORT, - ttyd_proxy_port=TTYD_PROXY_PORT, - ) + settings={ + "terminalEnabled": True, + "tunnelPorts": [], + "codeServerPort": CODE_SERVER_PORT, + "vncPort": NOVNC_PORT, + "terminalPort": TTYD_PROXY_PORT, + }, + ).resolve(sandbox, "sb-123") assert cs_url == "https://cs.example.com" assert vnc_url is None assert ttyd_url == "https://ttyd.example.com" @@ -169,12 +133,12 @@ class FakeSandbox: fake_create = MagicMock() fake_create.aio = fake_create_aio - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", fake_create) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", fake_create) monkeypatch.setattr( - SandboxManager, - "_resolve_and_setup_tunnels", - AsyncMock(return_value=(None, None, "https://ttyd.example.com", None)), + SandboxTunnels, + "resolve", + AsyncMock(return_value=TunnelUrls(None, None, "https://ttyd.example.com", None)), ) manager = SandboxManager() @@ -209,10 +173,10 @@ class FakeSandbox: fake_create = MagicMock() fake_create.aio = fake_create_aio - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", fake_create) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", fake_create) - tunnel_mock = AsyncMock(return_value=(None, None, None, None)) - monkeypatch.setattr(SandboxManager, "_resolve_and_setup_tunnels", tunnel_mock) + tunnel_mock = AsyncMock(return_value=TunnelUrls(None, None, None, None)) + monkeypatch.setattr(SandboxTunnels, "resolve", tunnel_mock) manager = SandboxManager() config = SandboxConfig( @@ -255,12 +219,14 @@ class FakeSandbox: fake_create = MagicMock() fake_create.aio = fake_create_aio - monkeypatch.setattr("src.sandbox.manager.modal.Image.from_id", fake_from_id) - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", fake_create) + monkeypatch.setattr("src.sandbox.launch.modal.Image.from_id", fake_from_id) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", fake_create) monkeypatch.setattr( - SandboxManager, - "_resolve_and_setup_tunnels", - AsyncMock(return_value=(None, None, "https://ttyd-restored.example.com", None)), + SandboxTunnels, + "resolve", + AsyncMock( + return_value=TunnelUrls(None, None, "https://ttyd-restored.example.com", None) + ), ) manager = SandboxManager() @@ -305,10 +271,10 @@ class FakeSandbox: fake_create = MagicMock() fake_create.aio = fake_create_aio - monkeypatch.setattr("src.sandbox.manager.modal.Image.from_id", fake_from_id) - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", fake_create) - tunnel_mock = AsyncMock(return_value=(None, None, None, None)) - monkeypatch.setattr(SandboxManager, "_resolve_and_setup_tunnels", tunnel_mock) + monkeypatch.setattr("src.sandbox.launch.modal.Image.from_id", fake_from_id) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", fake_create) + tunnel_mock = AsyncMock(return_value=TunnelUrls(None, None, None, None)) + monkeypatch.setattr(SandboxTunnels, "resolve", tunnel_mock) manager = SandboxManager() handle = await manager.restore_from_snapshot( diff --git a/packages/modal-infra/tests/test_tunnel_ports.py b/packages/modal-infra/tests/test_tunnel_ports.py index 68e51ab939..ead6c6b51a 100644 --- a/packages/modal-infra/tests/test_tunnel_ports.py +++ b/packages/modal-infra/tests/test_tunnel_ports.py @@ -14,6 +14,7 @@ TUNNEL_ENV_SANDBOX_ID_KEY, ) from src.sandbox.manager import CODE_SERVER_PORT, SandboxConfig, SandboxManager +from src.sandbox.tunnels import SandboxTunnels, TunnelUrls def _mock_sandbox_with_filesystem() -> tuple[MagicMock, AsyncMock]: @@ -27,7 +28,7 @@ def _mock_sandbox_with_filesystem() -> tuple[MagicMock, AsyncMock]: class TestResolveTunnels: - """SandboxManager._resolve_tunnels tests.""" + """SandboxTunnels._resolve_tunnels tests.""" @pytest.mark.asyncio async def test_resolves_all_ports(self): @@ -39,7 +40,7 @@ async def test_resolves_all_ports(self): sandbox = MagicMock() sandbox.tunnels.return_value = {3000: tunnel_3000, 3001: tunnel_3001} - result = await SandboxManager._resolve_tunnels(sandbox, "sb-1", [3000, 3001]) + result = await SandboxTunnels._resolve_tunnels(sandbox, "sb-1", [3000, 3001]) assert result == { 3000: "https://tunnel-3000.example.com", 3001: "https://tunnel-3001.example.com", @@ -53,9 +54,9 @@ async def test_returns_partial_on_missing_port(self): sandbox = MagicMock() sandbox.tunnels.return_value = {3000: tunnel_3000} - with patch("src.sandbox.manager.asyncio.sleep", new_callable=AsyncMock): - result = await SandboxManager._resolve_tunnels( - sandbox, "sb-1", [3000, 3001], retries=2, backoff=0.0 + with patch("src.sandbox.tunnels.asyncio.sleep", new_callable=AsyncMock): + result = await SandboxTunnels._resolve_tunnels( + sandbox, "sb-1", [3000, 3001], retries=2, backoff_seconds=0.0 ) assert result == {3000: "https://tunnel-3000.example.com"} @@ -64,9 +65,9 @@ async def test_returns_empty_on_exception_after_retries(self): sandbox = MagicMock() sandbox.tunnels.side_effect = Exception("tunnel unavailable") - with patch("src.sandbox.manager.asyncio.sleep", new_callable=AsyncMock): - result = await SandboxManager._resolve_tunnels( - sandbox, "sb-1", [3000], retries=3, backoff=0.0 + with patch("src.sandbox.tunnels.asyncio.sleep", new_callable=AsyncMock): + result = await SandboxTunnels._resolve_tunnels( + sandbox, "sb-1", [3000], retries=3, backoff_seconds=0.0 ) assert result == {} @@ -83,9 +84,9 @@ async def test_retries_on_partial_resolution(self): {3000: tunnel_3000, 3001: tunnel_3001}, ] - with patch("src.sandbox.manager.asyncio.sleep", new_callable=AsyncMock): - result = await SandboxManager._resolve_tunnels( - sandbox, "sb-1", [3000, 3001], retries=3, backoff=0.0 + with patch("src.sandbox.tunnels.asyncio.sleep", new_callable=AsyncMock): + result = await SandboxTunnels._resolve_tunnels( + sandbox, "sb-1", [3000, 3001], retries=3, backoff_seconds=0.0 ) assert result == { 3000: "https://tunnel-3000.example.com", @@ -100,17 +101,17 @@ class TestResolveAndSetupTunnels: @pytest.mark.asyncio async def test_returns_none_none_none_for_no_ports(self): sandbox = MagicMock() - cs_url, vnc_url, ttyd_url, extra = await SandboxManager._resolve_and_setup_tunnels( - sandbox, - "sb-1", - False, - False, - False, - [], - code_server_port=CODE_SERVER_PORT, - novnc_port=NOVNC_PORT, - ttyd_proxy_port=TTYD_PROXY_PORT, - ) + cs_url, vnc_url, ttyd_url, extra = await SandboxTunnels( + code_server_enabled=False, + vnc_enabled=False, + settings={ + "terminalEnabled": False, + "tunnelPorts": [], + "codeServerPort": CODE_SERVER_PORT, + "vncPort": NOVNC_PORT, + "terminalPort": TTYD_PROXY_PORT, + }, + ).resolve(sandbox, "sb-1") assert cs_url is None assert vnc_url is None assert ttyd_url is None @@ -122,22 +123,22 @@ async def test_resolves_extra_ports(self): sandbox, _write_text = _mock_sandbox_with_filesystem() with patch.object( - SandboxManager, + SandboxTunnels, "_resolve_tunnels", new_callable=AsyncMock, return_value=tunnel_urls, ): - cs_url, vnc_url, ttyd_url, extra = await SandboxManager._resolve_and_setup_tunnels( - sandbox, - "sb-1", - False, - False, - False, - [3000], - code_server_port=CODE_SERVER_PORT, - novnc_port=NOVNC_PORT, - ttyd_proxy_port=TTYD_PROXY_PORT, - ) + cs_url, vnc_url, ttyd_url, extra = await SandboxTunnels( + code_server_enabled=False, + vnc_enabled=False, + settings={ + "terminalEnabled": False, + "tunnelPorts": [3000], + "codeServerPort": CODE_SERVER_PORT, + "vncPort": NOVNC_PORT, + "terminalPort": TTYD_PROXY_PORT, + }, + ).resolve(sandbox, "sb-1") assert cs_url is None assert vnc_url is None @@ -154,22 +155,22 @@ async def test_splits_code_server_from_extra_ports(self): sandbox, _write_text = _mock_sandbox_with_filesystem() with patch.object( - SandboxManager, + SandboxTunnels, "_resolve_tunnels", new_callable=AsyncMock, return_value=resolved, ): - cs_url, vnc_url, ttyd_url, extra = await SandboxManager._resolve_and_setup_tunnels( - sandbox, - "sb-1", - True, - False, - False, - [3000], - code_server_port=CODE_SERVER_PORT, - novnc_port=NOVNC_PORT, - ttyd_proxy_port=TTYD_PROXY_PORT, - ) + cs_url, vnc_url, ttyd_url, extra = await SandboxTunnels( + code_server_enabled=True, + vnc_enabled=False, + settings={ + "terminalEnabled": False, + "tunnelPorts": [3000], + "codeServerPort": CODE_SERVER_PORT, + "vncPort": NOVNC_PORT, + "terminalPort": TTYD_PROXY_PORT, + }, + ).resolve(sandbox, "sb-1") assert cs_url == "https://cs.example.com" assert vnc_url is None @@ -183,22 +184,22 @@ async def test_keeps_code_server_port_tunnel_when_code_server_disabled(self): sandbox, _write_text = _mock_sandbox_with_filesystem() with patch.object( - SandboxManager, + SandboxTunnels, "_resolve_tunnels", new_callable=AsyncMock, return_value=resolved, ): - cs_url, vnc_url, ttyd_url, extra = await SandboxManager._resolve_and_setup_tunnels( - sandbox, - "sb-1", - False, - False, - False, - [CODE_SERVER_PORT], - code_server_port=CODE_SERVER_PORT, - novnc_port=NOVNC_PORT, - ttyd_proxy_port=TTYD_PROXY_PORT, - ) + cs_url, vnc_url, ttyd_url, extra = await SandboxTunnels( + code_server_enabled=False, + vnc_enabled=False, + settings={ + "terminalEnabled": False, + "tunnelPorts": [CODE_SERVER_PORT], + "codeServerPort": CODE_SERVER_PORT, + "vncPort": NOVNC_PORT, + "terminalPort": TTYD_PROXY_PORT, + }, + ).resolve(sandbox, "sb-1") assert cs_url is None assert vnc_url is None @@ -215,35 +216,35 @@ async def test_splits_custom_code_server_port_from_user_tunnel(self): sandbox, _write_text = _mock_sandbox_with_filesystem() with patch.object( - SandboxManager, + SandboxTunnels, "_resolve_tunnels", new_callable=AsyncMock, return_value=resolved, ): - cs_url, _vnc_url, _ttyd_url, extra = await SandboxManager._resolve_and_setup_tunnels( - sandbox, - "sb-1", - True, - False, - False, - [CODE_SERVER_PORT], - code_server_port=8081, - novnc_port=NOVNC_PORT, - ttyd_proxy_port=TTYD_PROXY_PORT, - ) + cs_url, _vnc_url, _ttyd_url, extra = await SandboxTunnels( + code_server_enabled=True, + vnc_enabled=False, + settings={ + "terminalEnabled": False, + "tunnelPorts": [CODE_SERVER_PORT], + "codeServerPort": 8081, + "vncPort": NOVNC_PORT, + "terminalPort": TTYD_PROXY_PORT, + }, + ).resolve(sandbox, "sb-1") assert cs_url == "https://cs.example.com" assert extra == {CODE_SERVER_PORT: "https://my-app.example.com"} class TestWriteTunnelEnvFile: - """SandboxManager._write_tunnel_env_file tests.""" + """SandboxTunnels._write_tunnel_env_file tests.""" @pytest.mark.asyncio async def test_writes_dotenv_format_to_expected_path(self): sandbox, write_text = _mock_sandbox_with_filesystem() - await SandboxManager._write_tunnel_env_file( + await SandboxTunnels._write_tunnel_env_file( sandbox, "sb-1", { @@ -267,8 +268,8 @@ async def test_write_failure_does_not_raise(self): sandbox, write_text = _mock_sandbox_with_filesystem() write_text.side_effect = Exception("write failed") - with patch("src.sandbox.manager.log") as mock_log: - await SandboxManager._write_tunnel_env_file( + with patch("src.sandbox.tunnels.log") as mock_log: + await SandboxTunnels._write_tunnel_env_file( sandbox, "sb-1", {3000: "https://tunnel-3000.example.com"} ) @@ -285,22 +286,22 @@ async def test_writes_file_when_extra_urls_present(self): tunnel_urls = {3000: "https://tunnel-3000.example.com"} with patch.object( - SandboxManager, + SandboxTunnels, "_resolve_tunnels", new_callable=AsyncMock, return_value=tunnel_urls, ): - await SandboxManager._resolve_and_setup_tunnels( - sandbox, - "sb-1", - False, - False, - False, - [3000], - code_server_port=CODE_SERVER_PORT, - novnc_port=NOVNC_PORT, - ttyd_proxy_port=TTYD_PROXY_PORT, - ) + await SandboxTunnels( + code_server_enabled=False, + vnc_enabled=False, + settings={ + "terminalEnabled": False, + "tunnelPorts": [3000], + "codeServerPort": CODE_SERVER_PORT, + "vncPort": NOVNC_PORT, + "terminalPort": TTYD_PROXY_PORT, + }, + ).resolve(sandbox, "sb-1") write_text.assert_awaited_once() written = write_text.call_args[0][0] @@ -312,22 +313,22 @@ async def test_does_not_write_file_when_no_extra_urls(self): sandbox, write_text = _mock_sandbox_with_filesystem() with patch.object( - SandboxManager, + SandboxTunnels, "_resolve_tunnels", new_callable=AsyncMock, return_value={}, ): - _cs, _vnc, _ttyd, extra = await SandboxManager._resolve_and_setup_tunnels( - sandbox, - "sb-1", - False, - False, - False, - [3000], - code_server_port=CODE_SERVER_PORT, - novnc_port=NOVNC_PORT, - ttyd_proxy_port=TTYD_PROXY_PORT, - ) + _cs, _vnc, _ttyd, extra = await SandboxTunnels( + code_server_enabled=False, + vnc_enabled=False, + settings={ + "terminalEnabled": False, + "tunnelPorts": [3000], + "codeServerPort": CODE_SERVER_PORT, + "vncPort": NOVNC_PORT, + "terminalPort": TTYD_PROXY_PORT, + }, + ).resolve(sandbox, "sb-1") assert extra is None write_text.assert_not_awaited() @@ -338,22 +339,22 @@ async def test_does_not_write_file_for_only_reserved_ports(self): sandbox, write_text = _mock_sandbox_with_filesystem() with patch.object( - SandboxManager, + SandboxTunnels, "_resolve_tunnels", new_callable=AsyncMock, return_value={CODE_SERVER_PORT: "https://cs.example.com"}, ): - await SandboxManager._resolve_and_setup_tunnels( - sandbox, - "sb-1", - True, - False, - False, - [], - code_server_port=CODE_SERVER_PORT, - novnc_port=NOVNC_PORT, - ttyd_proxy_port=TTYD_PROXY_PORT, - ) + await SandboxTunnels( + code_server_enabled=True, + vnc_enabled=False, + settings={ + "terminalEnabled": False, + "tunnelPorts": [], + "codeServerPort": CODE_SERVER_PORT, + "vncPort": NOVNC_PORT, + "terminalPort": TTYD_PROXY_PORT, + }, + ).resolve(sandbox, "sb-1") write_text.assert_not_awaited() @@ -364,24 +365,24 @@ async def test_write_failure_does_not_block_return(self): with ( patch.object( - SandboxManager, + SandboxTunnels, "_resolve_tunnels", new_callable=AsyncMock, return_value={3000: "https://tunnel-3000.example.com"}, ), - patch("src.sandbox.manager.log"), + patch("src.sandbox.tunnels.log"), ): - _cs, _vnc, _ttyd, extra = await SandboxManager._resolve_and_setup_tunnels( - sandbox, - "sb-1", - False, - False, - False, - [3000], - code_server_port=CODE_SERVER_PORT, - novnc_port=NOVNC_PORT, - ttyd_proxy_port=TTYD_PROXY_PORT, - ) + _cs, _vnc, _ttyd, extra = await SandboxTunnels( + code_server_enabled=False, + vnc_enabled=False, + settings={ + "terminalEnabled": False, + "tunnelPorts": [3000], + "codeServerPort": CODE_SERVER_PORT, + "vncPort": NOVNC_PORT, + "terminalPort": TTYD_PROXY_PORT, + }, + ).resolve(sandbox, "sb-1") assert extra == {3000: "https://tunnel-3000.example.com"} @@ -403,11 +404,11 @@ class FakeSandbox: return FakeSandbox() fake_create_aio.aio = fake_create_aio - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", fake_create_aio) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", fake_create_aio) monkeypatch.setattr( - SandboxManager, - "_resolve_and_setup_tunnels", - AsyncMock(return_value=(None, None, None, None)), + SandboxTunnels, + "resolve", + AsyncMock(return_value=TunnelUrls(None, None, None, None)), ) manager = SandboxManager() @@ -435,11 +436,11 @@ class FakeSandbox: return FakeSandbox() fake_create_aio.aio = fake_create_aio - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", fake_create_aio) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", fake_create_aio) monkeypatch.setattr( - SandboxManager, - "_resolve_and_setup_tunnels", - AsyncMock(return_value=(None, None, None, None)), + SandboxTunnels, + "resolve", + AsyncMock(return_value=TunnelUrls(None, None, None, None)), ) manager = SandboxManager() @@ -467,13 +468,13 @@ class FakeSandbox: fake_create_aio.aio = fake_create_aio monkeypatch.setattr( - "src.sandbox.manager.modal.Image.from_id", lambda *_a, **_kw: FakeImage() + "src.sandbox.launch.modal.Image.from_id", lambda *_a, **_kw: FakeImage() ) - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", fake_create_aio) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", fake_create_aio) monkeypatch.setattr( - SandboxManager, - "_resolve_and_setup_tunnels", - AsyncMock(return_value=(None, None, None, None)), + SandboxTunnels, + "resolve", + AsyncMock(return_value=TunnelUrls(None, None, None, None)), ) manager = SandboxManager() @@ -486,128 +487,61 @@ class FakeSandbox: assert captured["env"][EXPECTED_TUNNEL_PORTS_ENV_VAR] == "3000" -class TestCollectExposedPorts: - """SandboxManager._collect_exposed_ports tests.""" - - def test_no_ports_when_no_settings(self): - exposed, tunnel = SandboxManager._collect_exposed_ports( - False, False, False, None, CODE_SERVER_PORT, NOVNC_PORT, TTYD_PROXY_PORT - ) - assert exposed == [] - assert tunnel == [] - - def test_code_server_only(self): - exposed, tunnel = SandboxManager._collect_exposed_ports( - True, False, False, None, CODE_SERVER_PORT, NOVNC_PORT, TTYD_PROXY_PORT - ) - assert exposed == [CODE_SERVER_PORT] - assert tunnel == [] - - def test_tunnel_ports_only(self): - exposed, tunnel = SandboxManager._collect_exposed_ports( +@pytest.mark.parametrize( + "code_server, settings, exposed, expected_extras", + [ + (False, None, [], None), + (True, None, [CODE_SERVER_PORT], None), + (False, {"tunnelPorts": [3000, 5173]}, [3000, 5173], "3000,5173"), + (True, {"tunnelPorts": [3000]}, [CODE_SERVER_PORT, 3000], "3000"), + (False, {"terminalEnabled": True}, [TTYD_PROXY_PORT], None), + ( False, - False, - False, - {"tunnelPorts": [3000, 5173]}, - CODE_SERVER_PORT, - NOVNC_PORT, - TTYD_PROXY_PORT, - ) - assert exposed == [3000, 5173] - assert tunnel == [3000, 5173] - - def test_combined_code_server_and_tunnels(self): - exposed, tunnel = SandboxManager._collect_exposed_ports( + {"terminalEnabled": True, "tunnelPorts": [TTYD_PROXY_PORT, 3000]}, + [TTYD_PROXY_PORT, 3000], + "3000", + ), + (True, {"tunnelPorts": [CODE_SERVER_PORT, 3000]}, [CODE_SERVER_PORT, 3000], "3000"), + ( True, + {"codeServerPort": 8081, "tunnelPorts": [CODE_SERVER_PORT]}, + [8081, CODE_SERVER_PORT], + str(CODE_SERVER_PORT), + ), + ( False, - False, - {"tunnelPorts": [3000]}, - CODE_SERVER_PORT, - NOVNC_PORT, - TTYD_PROXY_PORT, - ) - assert exposed == [CODE_SERVER_PORT, 3000] - assert tunnel == [3000] - - def test_terminal_only(self): - exposed, tunnel = SandboxManager._collect_exposed_ports( - False, False, True, None, CODE_SERVER_PORT, NOVNC_PORT, TTYD_PROXY_PORT - ) - assert exposed == [TTYD_PROXY_PORT] - assert tunnel == [] - - def test_deduplicates_ttyd_port_from_tunnels(self): - exposed, tunnel = SandboxManager._collect_exposed_ports( - False, - False, - True, - {"tunnelPorts": [TTYD_PROXY_PORT, 3000]}, - CODE_SERVER_PORT, - NOVNC_PORT, - TTYD_PROXY_PORT, - ) - assert exposed == [TTYD_PROXY_PORT, 3000] - assert tunnel == [3000] - - def test_deduplicates_code_server_port_from_tunnels(self): - exposed, tunnel = SandboxManager._collect_exposed_ports( - True, - False, - False, - {"tunnelPorts": [CODE_SERVER_PORT, 3000]}, - CODE_SERVER_PORT, - NOVNC_PORT, - TTYD_PROXY_PORT, - ) - assert exposed == [CODE_SERVER_PORT, 3000] - assert tunnel == [3000] - - def test_custom_code_server_port_frees_default_for_tunnel(self): - # code-server moved to 8081 → the default 8080 is free as a user tunnel. - exposed, tunnel = SandboxManager._collect_exposed_ports( - True, - False, - False, - {"tunnelPorts": [CODE_SERVER_PORT]}, - 8081, - NOVNC_PORT, - TTYD_PROXY_PORT, - ) - assert exposed == [8081, CODE_SERVER_PORT] - assert tunnel == [CODE_SERVER_PORT] - - def test_custom_terminal_port_frees_default_for_tunnel(self): - exposed, tunnel = SandboxManager._collect_exposed_ports( - False, - False, - True, - {"tunnelPorts": [TTYD_PROXY_PORT, 3000]}, - CODE_SERVER_PORT, - NOVNC_PORT, - 7000, - ) - assert exposed == [7000, TTYD_PROXY_PORT, 3000] - assert tunnel == [TTYD_PROXY_PORT, 3000] + {"terminalEnabled": True, "terminalPort": 7000, "tunnelPorts": [TTYD_PROXY_PORT, 3000]}, + [7000, TTYD_PROXY_PORT, 3000], + f"{TTYD_PROXY_PORT},3000", + ), + ], +) +def test_exposed_ports_and_runtime_expectations_agree( + code_server, settings, exposed, expected_extras +): + tunnels = SandboxTunnels(code_server_enabled=code_server, settings=settings) + assert tunnels.exposed_ports == exposed + assert tunnels.environment.get(EXPECTED_TUNNEL_PORTS_ENV_VAR) == expected_extras class TestValidatePorts: - """SandboxManager._validate_ports tests.""" + """SandboxTunnels._validate_ports tests.""" def test_accepts_valid_ports(self): - assert SandboxManager._validate_ports([80, 3000, 65535]) == [80, 3000, 65535] + assert SandboxTunnels._validate_ports([80, 3000, 65535]) == [80, 3000, 65535] def test_rejects_out_of_range(self): - assert SandboxManager._validate_ports([0, -1, 65536, 3000]) == [3000] + assert SandboxTunnels._validate_ports([0, -1, 65536, 3000]) == [3000] def test_rejects_non_integers(self): - assert SandboxManager._validate_ports(["3000", 3.5, None, 8080]) == [8080] + assert SandboxTunnels._validate_ports(["3000", 3.5, None, 8080]) == [8080] def test_caps_at_ten(self): ports = list(range(1, 20)) - assert len(SandboxManager._validate_ports(ports)) == 10 + assert len(SandboxTunnels._validate_ports(ports)) == 10 def test_empty_list(self): - assert SandboxManager._validate_ports([]) == [] + assert SandboxTunnels._validate_ports([]) == [] def _patch_sandbox_create(monkeypatch, captured: dict) -> None: @@ -623,40 +557,40 @@ class FakeSandbox: return FakeSandbox() fake_create_aio.aio = fake_create_aio - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", fake_create_aio) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", fake_create_aio) monkeypatch.setattr( - SandboxManager, - "_resolve_and_setup_tunnels", - AsyncMock(return_value=(None, None, None, None)), + SandboxTunnels, + "resolve", + AsyncMock(return_value=TunnelUrls(None, None, None, None)), ) class TestResolveServicePorts: - """SandboxManager._resolve_service_ports tests.""" + """SandboxTunnels._resolve_service_ports tests.""" def test_defaults_when_unset(self): - assert SandboxManager._resolve_service_ports(None) == ( + assert SandboxTunnels._resolve_service_ports(None) == ( CODE_SERVER_PORT, NOVNC_PORT, TTYD_PROXY_PORT, ) - assert SandboxManager._resolve_service_ports({}) == ( + assert SandboxTunnels._resolve_service_ports({}) == ( CODE_SERVER_PORT, NOVNC_PORT, TTYD_PROXY_PORT, ) def test_uses_configured_ports(self): - assert SandboxManager._resolve_service_ports( + assert SandboxTunnels._resolve_service_ports( {"codeServerPort": 9000, "vncPort": 9001, "terminalPort": 9002} ) == (9000, 9001, 9002) def test_falls_back_on_invalid(self): - assert SandboxManager._resolve_service_ports( + assert SandboxTunnels._resolve_service_ports( {"codeServerPort": 0, "vncPort": -1, "terminalPort": 99999} ) == (CODE_SERVER_PORT, NOVNC_PORT, TTYD_PROXY_PORT) # strings and bools are not valid in-range ints - assert SandboxManager._resolve_service_ports( + assert SandboxTunnels._resolve_service_ports( {"codeServerPort": "8081", "vncPort": False, "terminalPort": True} ) == (CODE_SERVER_PORT, NOVNC_PORT, TTYD_PROXY_PORT) diff --git a/packages/modal-infra/tests/test_vm_resolve.py b/packages/modal-infra/tests/test_vm_resolve.py new file mode 100644 index 0000000000..a689d7fb28 --- /dev/null +++ b/packages/modal-infra/tests/test_vm_resolve.py @@ -0,0 +1,434 @@ +"""Generation-checked, lookup-only VM recovery API.""" + +import json +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest +from fastapi import HTTPException +from modal.exception import AlreadyExistsError, NotFoundError + +from sandbox_runtime.constants import ( + CODE_SERVER_PORT_ENV_VAR, + EXPECTED_TUNNEL_PORTS_ENV_VAR, + NOVNC_PORT_ENV_VAR, + TTYD_PROXY_PORT_ENV_VAR, + VNC_PASSWORD_ENV_VAR, +) +from src import web_api +from src.sandbox import manager as manager_module +from src.sandbox.launch_policy import docker_allocation_name, docker_allocation_tags +from src.sandbox.tunnels import SandboxTunnels, TunnelUrls + +SESSION = "session-1" +GENERATION = "generation-1" +RESOLVE_REQUEST = {"session_id": SESSION, "sandbox_id": GENERATION} + + +def _tags(launch="1-111-9000-9001-9002", ports="3000-3001"): + return { + **docker_allocation_tags(SESSION, GENERATION), + "openinspect_vm_launch": launch, + "openinspect_vm_ports": ports, + } + + +async def _call(endpoint, request, authorization="Bearer test"): + return await endpoint.get_raw_f()( + request, + authorization=authorization, + x_trace_id=None, + x_request_id=None, + x_session_id=None, + x_sandbox_id=None, + ) + + +def _sandbox(tags, env=None): + async def execute(*args, **kwargs): + keys = args[4:] + output = json.dumps({key: (env or {}).get(key) for key in keys}) + return SimpleNamespace( + stdout=SimpleNamespace(read=SimpleNamespace(aio=AsyncMock(return_value=output))), + wait=SimpleNamespace(aio=AsyncMock(return_value=0)), + ) + + return SimpleNamespace( + object_id="sb-real-id", + get_tags=SimpleNamespace(aio=AsyncMock(return_value=tags)), + exec=SimpleNamespace(aio=AsyncMock(side_effect=execute)), + terminate=SimpleNamespace(aio=AsyncMock()), + ) + + +@pytest.mark.asyncio +async def test_resolve_returns_owned_vm_id_access_and_tunnels_without_mutation(monkeypatch): + monkeypatch.setattr(web_api, "require_auth", lambda _token: None) + env = { + "CODE_SERVER_PASSWORD": "original-code-password", + VNC_PASSWORD_ENV_VAR: "original-vnc-password", + CODE_SERVER_PORT_ENV_VAR: "9000", + NOVNC_PORT_ENV_VAR: "9001", + TTYD_PROXY_PORT_ENV_VAR: "9002", + EXPECTED_TUNNEL_PORTS_ENV_VAR: "3000,3001", + "TERMINAL_ENABLED": "true", + } + sandbox = _sandbox(_tags(), env) + from_name = AsyncMock(return_value=sandbox) + create = AsyncMock(side_effect=AssertionError("resolve must not create")) + monkeypatch.setattr(manager_module.modal.Sandbox, "from_name", SimpleNamespace(aio=from_name)) + monkeypatch.setattr(manager_module.modal.Sandbox, "create", SimpleNamespace(aio=create)) + tunnels = AsyncMock( + return_value=TunnelUrls( + "https://code.example", + "https://vnc.example", + "https://terminal.example", + {3000: "https://app.example", 3001: "https://other.example"}, + ) + ) + monkeypatch.setattr(SandboxTunnels, "resolve", tunnels) + + result = await _call(web_api.api_resolve_vm_sandbox, RESOLVE_REQUEST) + + assert result == { + "success": True, + "data": { + "sandbox_id": GENERATION, + "modal_object_id": "sb-real-id", + "code_server_url": "https://code.example", + "code_server_password": "original-code-password", + "vnc_url": "https://vnc.example", + "vnc_password": "original-vnc-password", + "ttyd_url": "https://terminal.example", + "tunnel_urls": {3000: "https://app.example", 3001: "https://other.example"}, + "sandbox_backend": "modal-vm", + }, + } + from_name.assert_awaited_once_with("open-inspect", docker_allocation_name(SESSION)) + sandbox.get_tags.aio.assert_awaited_once_with() + sandbox.exec.aio.assert_awaited_once() + sandbox.terminate.aio.assert_not_awaited() + create.assert_not_awaited() + tunnels.assert_awaited_once_with(sandbox, GENERATION, write_env_file=False) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("missing_port", [9000, 9001, 9002, 3001]) +async def test_resolve_retries_when_enabled_tunnel_is_missing(monkeypatch, missing_port): + monkeypatch.setattr(web_api, "require_auth", lambda _token: None) + sandbox = _sandbox( + _tags(), + { + "CODE_SERVER_PASSWORD": "original-code-password", + VNC_PASSWORD_ENV_VAR: "original-vnc-password", + CODE_SERVER_PORT_ENV_VAR: "9000", + NOVNC_PORT_ENV_VAR: "9001", + TTYD_PROXY_PORT_ENV_VAR: "9002", + EXPECTED_TUNNEL_PORTS_ENV_VAR: "3000,3001", + "TERMINAL_ENABLED": "true", + }, + ) + monkeypatch.setattr( + manager_module.modal.Sandbox, + "from_name", + SimpleNamespace(aio=AsyncMock(return_value=sandbox)), + ) + create = AsyncMock(side_effect=AssertionError("resolve must not create")) + monkeypatch.setattr(manager_module.modal.Sandbox, "create", SimpleNamespace(aio=create)) + monkeypatch.setattr( + SandboxTunnels, + "_resolve_tunnels", + AsyncMock( + return_value={ + port: f"https://port-{port}.example" + for port in [9000, 9001, 9002, 3000, 3001] + if port != missing_port + } + ), + ) + write_env = AsyncMock(side_effect=AssertionError("resolve must not write")) + monkeypatch.setattr(SandboxTunnels, "_write_tunnel_env_file", write_env) + + with pytest.raises(HTTPException) as exc: + await _call(web_api.api_resolve_vm_sandbox, RESOLVE_REQUEST) + + assert (exc.value.status_code, exc.value.detail) == (409, "race_pending") + create.assert_not_awaited() + sandbox.terminate.aio.assert_not_awaited() + write_env.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_resolve_disabled_access_does_not_return_credentials(monkeypatch): + monkeypatch.setattr(web_api, "require_auth", lambda _token: None) + sandbox = _sandbox(_tags("1-000-8080-6080-7680", "none")) + monkeypatch.setattr( + manager_module.modal.Sandbox, + "from_name", + SimpleNamespace(aio=AsyncMock(return_value=sandbox)), + ) + tunnels = AsyncMock(return_value=TunnelUrls()) + monkeypatch.setattr(SandboxTunnels, "resolve", tunnels) + + result = await _call(web_api.api_resolve_vm_sandbox, RESOLVE_REQUEST) + + assert result["data"]["code_server_password"] is None + assert result["data"]["vnc_password"] is None + tunnels.assert_awaited_once_with(sandbox, GENERATION, write_env_file=False) + + +@pytest.mark.asyncio +async def test_resolve_extra_tunnels_does_not_write_into_vm(monkeypatch): + monkeypatch.setattr(web_api, "require_auth", lambda _token: None) + sandbox = _sandbox( + _tags("1-000-8080-6080-7680", "3000"), + {EXPECTED_TUNNEL_PORTS_ENV_VAR: "9999"}, + ) + monkeypatch.setattr( + manager_module.modal.Sandbox, + "from_name", + SimpleNamespace(aio=AsyncMock(return_value=sandbox)), + ) + monkeypatch.setattr( + SandboxTunnels, + "_resolve_tunnels", + AsyncMock(return_value={3000: "https://app.example"}), + ) + write_env = AsyncMock(side_effect=AssertionError("resolve must not write")) + monkeypatch.setattr(SandboxTunnels, "_write_tunnel_env_file", write_env) + + result = await _call(web_api.api_resolve_vm_sandbox, RESOLVE_REQUEST) + + assert result["data"]["tunnel_urls"] == {3000: "https://app.example"} + write_env.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_resolve_legacy_user_password_does_not_enable_access(monkeypatch): + monkeypatch.setattr(web_api, "require_auth", lambda _token: None) + sandbox = _sandbox( + docker_allocation_tags(SESSION, GENERATION), + { + "CODE_SERVER_PASSWORD": "user-repo-secret", + VNC_PASSWORD_ENV_VAR: "user-vnc-secret", + CODE_SERVER_PORT_ENV_VAR: "9000", + EXPECTED_TUNNEL_PORTS_ENV_VAR: "3000", + "TERMINAL_ENABLED": "true", + }, + ) + monkeypatch.setattr( + manager_module.modal.Sandbox, + "from_name", + SimpleNamespace(aio=AsyncMock(return_value=sandbox)), + ) + tunnels = AsyncMock(side_effect=AssertionError("legacy resolve must not inspect tunnels")) + monkeypatch.setattr(SandboxTunnels, "resolve", tunnels) + + result = await _call(web_api.api_resolve_vm_sandbox, RESOLVE_REQUEST) + + assert result["data"] == { + "sandbox_id": GENERATION, + "modal_object_id": "sb-real-id", + "code_server_url": None, + "code_server_password": None, + "vnc_url": None, + "vnc_password": None, + "ttyd_url": None, + "tunnel_urls": None, + "sandbox_backend": "modal-vm", + } + sandbox.exec.aio.assert_not_awaited() + sandbox.terminate.aio.assert_not_awaited() + tunnels.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_resolve_versioned_flags_ignore_user_password_when_service_disabled(monkeypatch): + monkeypatch.setattr(web_api, "require_auth", lambda _token: None) + sandbox = _sandbox( + _tags("1-010-9000-9001-9002", "none"), + {"CODE_SERVER_PASSWORD": "user-repo-secret", VNC_PASSWORD_ENV_VAR: "vnc-password"}, + ) + monkeypatch.setattr( + manager_module.modal.Sandbox, + "from_name", + SimpleNamespace(aio=AsyncMock(return_value=sandbox)), + ) + tunnels = AsyncMock(return_value=TunnelUrls(vnc_url="https://vnc.example")) + monkeypatch.setattr(SandboxTunnels, "resolve", tunnels) + + result = await _call(web_api.api_resolve_vm_sandbox, RESOLVE_REQUEST) + + assert result["data"]["code_server_password"] is None + assert result["data"]["code_server_url"] is None + assert result["data"]["vnc_password"] == "vnc-password" + assert sandbox.exec.aio.call_args.args[-1:] == (VNC_PASSWORD_ENV_VAR,) + tunnels.assert_awaited_once_with(sandbox, GENERATION, write_env_file=False) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "launch,ports", + [ + ("2-111-9000-9001-9002", "3000"), + ("1-111-9000-9001-9002", None), + ("1-111-0-9001-9002", "3000"), + ("1-11x-9000-9001-9002", "3000"), + ("1-111-9000-9001-9002", "65536"), + ], +) +async def test_resolve_unknown_or_incomplete_metadata_never_falls_back_to_env( + monkeypatch, launch, ports +): + monkeypatch.setattr(web_api, "require_auth", lambda _token: None) + tags = _tags(launch, ports) + if ports is None: + del tags["openinspect_vm_ports"] + sandbox = _sandbox(tags, {"CODE_SERVER_PASSWORD": "user-repo-secret"}) + monkeypatch.setattr( + manager_module.modal.Sandbox, + "from_name", + SimpleNamespace(aio=AsyncMock(return_value=sandbox)), + ) + tunnels = AsyncMock(side_effect=AssertionError("invalid metadata must not inspect tunnels")) + monkeypatch.setattr(SandboxTunnels, "resolve", tunnels) + + result = await _call(web_api.api_resolve_vm_sandbox, RESOLVE_REQUEST) + + assert result["data"]["modal_object_id"] == "sb-real-id" + assert result["data"]["code_server_password"] is None + assert result["data"]["tunnel_urls"] is None + sandbox.exec.aio.assert_not_awaited() + tunnels.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("allocation", "status", "detail"), + [ + (None, 409, "not_visible"), + ("foreign", 409, "other_generation"), + ], +) +async def test_resolve_reports_typed_absence_or_foreign_generation( + monkeypatch, allocation, status, detail +): + monkeypatch.setattr(web_api, "require_auth", lambda _token: None) + lookup = ( + AsyncMock(side_effect=NotFoundError("not visible")) + if allocation is None + else AsyncMock(return_value=_sandbox(docker_allocation_tags(SESSION, "other"))) + ) + monkeypatch.setattr(manager_module.modal.Sandbox, "from_name", SimpleNamespace(aio=lookup)) + create = AsyncMock() + monkeypatch.setattr(manager_module.modal.Sandbox, "create", SimpleNamespace(aio=create)) + + with pytest.raises(HTTPException) as exc: + await _call(web_api.api_resolve_vm_sandbox, RESOLVE_REQUEST) + + assert (exc.value.status_code, exc.value.detail) == (status, detail) + create.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_resolve_rejects_unexpected_allocation_tags(monkeypatch): + monkeypatch.setattr(web_api, "require_auth", lambda _token: None) + sandbox = _sandbox({**_tags(), "unexpected": "tag"}) + monkeypatch.setattr( + manager_module.modal.Sandbox, + "from_name", + SimpleNamespace(aio=AsyncMock(return_value=sandbox)), + ) + + with pytest.raises(HTTPException) as exc: + await _call(web_api.api_resolve_vm_sandbox, RESOLVE_REQUEST) + + assert (exc.value.status_code, exc.value.detail) == (409, "other_generation") + sandbox.exec.aio.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_resolve_authenticates_before_lookup_or_validation(monkeypatch): + lookup = AsyncMock() + monkeypatch.setattr(manager_module.modal.Sandbox, "from_name", SimpleNamespace(aio=lookup)) + monkeypatch.setattr( + web_api, + "require_auth", + lambda _token: (_ for _ in ()).throw(HTTPException(status_code=401)), + ) + + with pytest.raises(HTTPException) as exc: + await _call(web_api.api_resolve_vm_sandbox, {"sandbox_auth_token": "secret"}, None) + + assert exc.value.status_code == 401 + lookup.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "extra", ["sandbox_auth_token", "user_env_vars", "retire_sandbox_id", "control_plane_url"] +) +async def test_resolve_rejects_secret_or_mutating_request_fields(monkeypatch, extra): + monkeypatch.setattr(web_api, "require_auth", lambda _token: None) + lookup = AsyncMock() + monkeypatch.setattr(manager_module.modal.Sandbox, "from_name", SimpleNamespace(aio=lookup)) + + with pytest.raises(HTTPException) as exc: + await _call(web_api.api_resolve_vm_sandbox, {**RESOLVE_REQUEST, extra: "forbidden"}) + + assert exc.value.status_code == 400 + lookup.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("field", ["session_id", "sandbox_id"]) +async def test_resolve_requires_both_identity_fields(monkeypatch, field): + monkeypatch.setattr(web_api, "require_auth", lambda _token: None) + + with pytest.raises(HTTPException) as exc: + await _call( + web_api.api_resolve_vm_sandbox, {k: v for k, v in RESOLVE_REQUEST.items() if k != field} + ) + + assert exc.value.status_code == 400 + assert exc.value.detail == f"{field} is required" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("endpoint", ["api_create_sandbox", "api_restore_sandbox"]) +@pytest.mark.parametrize( + ("case", "detail"), + [("foreign", "other_generation"), ("expired", "window_closed"), ("race", "race_pending")], +) +async def test_vm_launch_reports_typed_outcomes(monkeypatch, endpoint, case, detail): + monkeypatch.setattr(web_api, "require_auth", lambda _token: None) + monkeypatch.setattr(web_api, "require_valid_control_plane_url", lambda _url: None) + monkeypatch.setattr("src.images.base.docker_image", object()) + monkeypatch.setattr(manager_module.modal.Image, "from_id", lambda _id: object()) + lookup = ( + AsyncMock(return_value=_sandbox(docker_allocation_tags(SESSION, "other"))) + if case == "foreign" + else AsyncMock(side_effect=NotFoundError("not visible")) + ) + create = AsyncMock(side_effect=AlreadyExistsError("winner not visible")) + monkeypatch.setattr(manager_module.modal.Sandbox, "from_name", SimpleNamespace(aio=lookup)) + monkeypatch.setattr(manager_module.modal.Sandbox, "create", SimpleNamespace(aio=create)) + request = { + "sandbox_id": GENERATION, + "control_plane_url": "https://control.example", + "sandbox_auth_token": "secret", + "sandbox_backend": "modal-vm", + "launch_deadline_at_ms": 1 if case == "expired" else 9999999999999, + } + if endpoint == "api_create_sandbox": + request["session_id"] = SESSION + else: + request["session_config"] = {"session_id": SESSION} + request["snapshot_image_id"] = "im-snapshot" + + with pytest.raises(HTTPException) as exc: + await _call(getattr(web_api, endpoint), request) + + assert (exc.value.status_code, exc.value.detail) == (409, detail) + assert create.await_count == (1 if case == "race" else 0) + assert lookup.await_count == (2 if case == "race" else 1) diff --git a/packages/modal-infra/tests/test_vnc.py b/packages/modal-infra/tests/test_vnc.py index cca7044ac6..18e2412789 100644 --- a/packages/modal-infra/tests/test_vnc.py +++ b/packages/modal-infra/tests/test_vnc.py @@ -5,6 +5,7 @@ import pytest from sandbox_runtime.constants import ( + EXPECTED_TUNNEL_PORTS_ENV_VAR, NOVNC_PORT, NOVNC_PORT_ENV_VAR, VNC_PASSWORD_ENV_VAR, @@ -12,6 +13,7 @@ VNC_PORT, ) from src.sandbox.manager import CODE_SERVER_PORT, TTYD_PROXY_PORT, SandboxConfig, SandboxManager +from src.sandbox.tunnels import SandboxTunnels, TunnelUrls def _patch_sandbox_create(monkeypatch, captured: dict) -> None: @@ -27,7 +29,7 @@ class FakeSandbox: fake_create = MagicMock() fake_create.aio = fake_create_aio - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", fake_create) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", fake_create) class TestCreateSandboxVnc: @@ -36,9 +38,9 @@ async def test_returns_url_and_password_and_exposes_only_novnc(self, monkeypatch captured = {} _patch_sandbox_create(monkeypatch, captured) monkeypatch.setattr( - SandboxManager, - "_resolve_and_setup_tunnels", - AsyncMock(return_value=(None, "https://vnc.example.com", None, None)), + SandboxTunnels, + "resolve", + AsyncMock(return_value=TunnelUrls(None, "https://vnc.example.com", None, None)), ) handle = await SandboxManager().create_sandbox( @@ -63,9 +65,9 @@ async def test_disabled_vnc_has_no_credentials_or_port(self, monkeypatch): captured = {} _patch_sandbox_create(monkeypatch, captured) monkeypatch.setattr( - SandboxManager, - "_resolve_and_setup_tunnels", - AsyncMock(return_value=(None, None, None, None)), + SandboxTunnels, + "resolve", + AsyncMock(return_value=TunnelUrls(None, None, None, None)), ) handle = await SandboxManager().create_sandbox( @@ -84,11 +86,13 @@ class TestRestoreSandboxVnc: async def test_generates_credentials_and_returns_them_with_url(self, monkeypatch): captured = {} _patch_sandbox_create(monkeypatch, captured) - monkeypatch.setattr("src.sandbox.manager.modal.Image.from_id", lambda *_args: MagicMock()) + monkeypatch.setattr("src.sandbox.launch.modal.Image.from_id", lambda *_args: MagicMock()) monkeypatch.setattr( - SandboxManager, - "_resolve_and_setup_tunnels", - AsyncMock(return_value=(None, "https://restored-vnc.example.com", None, None)), + SandboxTunnels, + "resolve", + AsyncMock( + return_value=TunnelUrls(None, "https://restored-vnc.example.com", None, None) + ), ) handle = await SandboxManager().restore_from_snapshot( @@ -108,37 +112,28 @@ async def test_generates_credentials_and_returns_them_with_url(self, monkeypatch async def test_resolves_custom_novnc_tunnel(): sandbox = MagicMock() with patch.object( - SandboxManager, + SandboxTunnels, "_resolve_tunnels", new_callable=AsyncMock, return_value={6081: "https://vnc.example.com"}, ) as resolve_tunnels: - result = await SandboxManager._resolve_and_setup_tunnels( - sandbox, - "sandbox-vnc", - False, - True, - False, - [], - code_server_port=CODE_SERVER_PORT, - novnc_port=6081, - ttyd_proxy_port=TTYD_PROXY_PORT, - ) + result = await SandboxTunnels( + code_server_enabled=False, + vnc_enabled=True, + settings={ + "terminalEnabled": False, + "tunnelPorts": [], + "codeServerPort": CODE_SERVER_PORT, + "vncPort": 6081, + "terminalPort": TTYD_PROXY_PORT, + }, + ).resolve(sandbox, "sandbox-vnc") resolve_tunnels.assert_awaited_once_with(sandbox, "sandbox-vnc", [6081]) assert result == (None, "https://vnc.example.com", None, None) def test_raw_vnc_port_is_never_exposed_as_an_extra_tunnel(): - exposed, extras = SandboxManager._collect_exposed_ports( - False, - False, - False, - {"tunnelPorts": [VNC_PORT, 3000]}, - CODE_SERVER_PORT, - NOVNC_PORT, - TTYD_PROXY_PORT, - ) - - assert exposed == [3000] - assert extras == [3000] + tunnels = SandboxTunnels(settings={"tunnelPorts": [VNC_PORT, 3000]}) + assert tunnels.exposed_ports == [3000] + assert tunnels.environment[EXPECTED_TUNNEL_PORTS_ENV_VAR] == "3000" diff --git a/packages/modal-infra/uv.lock b/packages/modal-infra/uv.lock index af07c319f3..48aa33c884 100644 --- a/packages/modal-infra/uv.lock +++ b/packages/modal-infra/uv.lock @@ -331,7 +331,7 @@ wheels = [ [[package]] name = "claude-agent-sdk" -version = "0.2.158" +version = "0.2.161" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, @@ -339,13 +339,12 @@ dependencies = [ { name = "mcp" }, { name = "sniffio" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/0e/5c/1d348e677fb199978d2b559fb5822fc58d77927f0988d1eb2acc18d5a0bb/claude_agent_sdk-0.2.158.tar.gz", hash = "sha256:672ddb19df334e9cb80bb264a6ddf00870c8d4e170b3fd0a3cf0ce354f634e7f", size = 354570, upload-time = "2026-09-23T01:39:14.77Z" } +sdist = { url = "https://files.pythonhosted.org/packages/8b/e1/8c94ff7f200eabcee9a9e57825dfac596bb6fbdeeafa594f49bc7e7f89b5/claude_agent_sdk-0.2.161.tar.gz", hash = "sha256:284a1372ec3af133578009ab2b821a238272dbf17d687a58be9df918d7b3ae89", size = 365168, upload-time = "2026-09-28T18:53:39.197Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/89/ad/37559b5a12170699eb6d4c5e1c66f491c4a3d7366f8f300a8dba0fc40aec/claude_agent_sdk-0.2.158-py3-none-macosx_11_0_arm64.whl", hash = "sha256:d124bd374dfbbcbdb4326e922f30d436251ae469df9cae545ef8ff6839d76e47", size = 90406616, upload-time = "2026-09-23T01:39:18.829Z" }, - { url = "https://files.pythonhosted.org/packages/c1/52/21968fc645ee206bb260f18b26ae8ae035fc5af820a7820b01e308496510/claude_agent_sdk-0.2.158-py3-none-macosx_11_0_x86_64.whl", hash = "sha256:beb963aa66eac715ff005e44ff02065fd5d855a10a47bc802097a31a522e6be3", size = 94751713, upload-time = "2026-09-23T01:39:22.591Z" }, - { url = "https://files.pythonhosted.org/packages/a1/4b/b42ab1a7ba32cea66184dedf8da158d49d00d6af753bd48998dab6e7a1d6/claude_agent_sdk-0.2.158-py3-none-manylinux_2_17_aarch64.whl", hash = "sha256:12f432273c3f0da103ea8dc921c3ce96e802c044e1f3f2535cf98b7d59777aa7", size = 99897552, upload-time = "2026-09-23T01:39:26.933Z" }, - { url = "https://files.pythonhosted.org/packages/e3/eb/1b34083c59a09a1a380c211eacd1cb185de4adc4bdb1170997e37f53f6ca/claude_agent_sdk-0.2.158-py3-none-manylinux_2_17_x86_64.whl", hash = "sha256:1f46ddac2cb51f16b2ca064a0dbeeab7db712f34e72454cc7776713e0f0f663c", size = 100257803, upload-time = "2026-09-23T01:39:31.501Z" }, - { url = "https://files.pythonhosted.org/packages/15/0d/11ab53934b04176180099a79bb186036e304e1fe1310bdecf21df599f1d3/claude_agent_sdk-0.2.158-py3-none-win_amd64.whl", hash = "sha256:b1e7f6b9d048c2c44e5228d6a0eeb19ae3aafb643c1115c456ea9937e0439aef", size = 102853711, upload-time = "2026-09-23T01:39:36.133Z" }, + { url = "https://files.pythonhosted.org/packages/e8/be/528e3ec2bb30deff4fcc9484dd47d1022786ad3659d31110bd0ea66ea548/claude_agent_sdk-0.2.161-py3-none-macosx_11_0_arm64.whl", hash = "sha256:127c18a790dc9f77cc2d648f932e11b06cd3325c4b3de7a73d3bcf3d3b74b8bc", size = 93872213, upload-time = "2026-09-28T18:53:42.956Z" }, + { url = "https://files.pythonhosted.org/packages/17/1c/a0dcb628ac2f25bc92d858857a399202723098057828a909ca4f9893b721/claude_agent_sdk-0.2.161-py3-none-macosx_11_0_x86_64.whl", hash = "sha256:599e1c8ccb4d48ed57999b48e80937d6a6aefbc52b0c0cda1291aea21df29e3e", size = 98316495, upload-time = "2026-09-28T18:53:47.239Z" }, + { url = "https://files.pythonhosted.org/packages/73/04/3c0caee46c27eee75642b52ee11857c0d3b9d6917ecfd916da2ffc2c2d3f/claude_agent_sdk-0.2.161-py3-none-manylinux_2_17_aarch64.whl", hash = "sha256:18081d3277c07a71891bd15d1496387d9efc5f04a6b482ae3a03e7a0b8067069", size = 103331951, upload-time = "2026-09-28T18:53:51.367Z" }, + { url = "https://files.pythonhosted.org/packages/09/70/2c7c4160a19c6edd42e317bdc9cf810e91b6a535c04377fb09d4ee090769/claude_agent_sdk-0.2.161-py3-none-manylinux_2_17_x86_64.whl", hash = "sha256:b854e46d2889a51e1a77978711a05c757e63064bd5d8f89bcf74bb507babfc2c", size = 103684983, upload-time = "2026-09-28T18:53:55.496Z" }, ] [[package]] @@ -1060,7 +1059,7 @@ dev = [ requires-dist = [ { name = "fastapi", specifier = ">=0.110.0" }, { name = "httpx", specifier = ">=0.27.0" }, - { name = "modal", specifier = ">=1.4.3" }, + { name = "modal", specifier = "==1.4.3" }, { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.14.0" }, { name = "open-inspect-sandbox-runtime", editable = "../sandbox-runtime" }, { name = "pydantic", specifier = ">=2.0" }, @@ -1086,7 +1085,7 @@ dependencies = [ [package.metadata] requires-dist = [ - { name = "claude-agent-sdk", specifier = "==0.2.158" }, + { name = "claude-agent-sdk", specifier = "==0.2.161" }, { name = "cryptography", specifier = ">=44.0.0" }, { name = "httpx", specifier = ">=0.27.0" }, { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.14.0" }, diff --git a/packages/sandbox-images/locks/runtime.txt b/packages/sandbox-images/locks/runtime.txt index 8f6978407f..d20a2d4dbf 100644 --- a/packages/sandbox-images/locks/runtime.txt +++ b/packages/sandbox-images/locks/runtime.txt @@ -86,13 +86,12 @@ cffi==2.1.1 ; platform_python_implementation != 'PyPy' \ --hash=sha256:f81b3b8f3d4e343550fa4baa0e479bba9f2d29ce9c2e9b51d1ce1718d7442fcf \ --hash=sha256:fb92203a88b3d3053034db775110081c49d28be6551923805e039924093761e4 \ --hash=sha256:fcd22650c908d7b7da162bbfaab594a1227a15d1643a98c68b122ac642fa2264 -claude-agent-sdk==0.2.158 \ - --hash=sha256:12f432273c3f0da103ea8dc921c3ce96e802c044e1f3f2535cf98b7d59777aa7 \ - --hash=sha256:1f46ddac2cb51f16b2ca064a0dbeeab7db712f34e72454cc7776713e0f0f663c \ - --hash=sha256:672ddb19df334e9cb80bb264a6ddf00870c8d4e170b3fd0a3cf0ce354f634e7f \ - --hash=sha256:b1e7f6b9d048c2c44e5228d6a0eeb19ae3aafb643c1115c456ea9937e0439aef \ - --hash=sha256:beb963aa66eac715ff005e44ff02065fd5d855a10a47bc802097a31a522e6be3 \ - --hash=sha256:d124bd374dfbbcbdb4326e922f30d436251ae469df9cae545ef8ff6839d76e47 +claude-agent-sdk==0.2.161 \ + --hash=sha256:127c18a790dc9f77cc2d648f932e11b06cd3325c4b3de7a73d3bcf3d3b74b8bc \ + --hash=sha256:18081d3277c07a71891bd15d1496387d9efc5f04a6b482ae3a03e7a0b8067069 \ + --hash=sha256:284a1372ec3af133578009ab2b821a238272dbf17d687a58be9df918d7b3ae89 \ + --hash=sha256:599e1c8ccb4d48ed57999b48e80937d6a6aefbc52b0c0cda1291aea21df29e3e \ + --hash=sha256:b854e46d2889a51e1a77978711a05c757e63064bd5d8f89bcf74bb507babfc2c click==8.5.0 ; sys_platform != 'emscripten' \ --hash=sha256:255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360 \ --hash=sha256:ba0d2089de75ea0310e2dde03160e6ca10009947fb95a182f9b54021bb272e34 diff --git a/packages/sandbox-runtime/pyproject.toml b/packages/sandbox-runtime/pyproject.toml index 5325d57fb3..6bf50edd44 100644 --- a/packages/sandbox-runtime/pyproject.toml +++ b/packages/sandbox-runtime/pyproject.toml @@ -11,7 +11,7 @@ dependencies = [ "PyJWT[crypto]>=2.9.0", # Exact pin: the wheel bundles the `claude` binary and its message shapes # are what harness/claude.py translates. - "claude-agent-sdk==0.2.158", + "claude-agent-sdk==0.2.161", ] [project.optional-dependencies] diff --git a/packages/sandbox-runtime/src/sandbox_runtime/plugins/codex-auth-plugin.js b/packages/sandbox-runtime/src/sandbox_runtime/plugins/codex-auth-plugin.js index 9d15f0dff5..ba9c24877b 100644 --- a/packages/sandbox-runtime/src/sandbox_runtime/plugins/codex-auth-plugin.js +++ b/packages/sandbox-runtime/src/sandbox_runtime/plugins/codex-auth-plugin.js @@ -626,6 +626,8 @@ export const CodexAuthProxy = async (input) => { const { accessToken, accountId } = await ensureAccessToken(getAuth, setAuth); proxied.headers.set("authorization", `Bearer ${accessToken}`); if (accountId) proxied.headers.set("ChatGPT-Account-Id", accountId); + // No generation stream here, so OpenCode's stream timeout does not + // bound it: Bun's fetch timeout stays in force. return fetch(proxied); } @@ -640,7 +642,9 @@ export const CodexAuthProxy = async (input) => { const overrides = Object.fromEntries( Object.entries(restInit).filter(([, value]) => value !== undefined) ); - const baseInit = { ...inherited, ...overrides, method, body, signal }; + // Let OpenCode's stream timeout handle stalls instead of Bun's fetch idle + // timer, on the subscription and the fallback path alike. + const baseInit = { ...inherited, ...overrides, method, body, signal, timeout: false }; // opencode signs the request with a placeholder API key; this proxy // supplies the real credential instead. diff --git a/packages/sandbox-runtime/tests/codex-auth-plugin.test.mjs b/packages/sandbox-runtime/tests/codex-auth-plugin.test.mjs index bd11a784de..aa72f984f3 100644 --- a/packages/sandbox-runtime/tests/codex-auth-plugin.test.mjs +++ b/packages/sandbox-runtime/tests/codex-auth-plugin.test.mjs @@ -856,6 +856,7 @@ test("dispatches a non-generation Request without reshaping it", async () => { assert.equal(call.request.headers.get("authorization"), "Bearer cp-access"); const sent = new Uint8Array(await call.request.arrayBuffer()); assert.deepEqual([...sent], [...body], "the body bytes are untouched"); + assert.equal(call.init.timeout, undefined, "Bun's fetch timeout stays in force"); }); test("cancels a stalled streaming body when the caller aborts", async () => { @@ -1050,6 +1051,7 @@ async function withCatalog(models, run) { test("preserves a source Request while proxying Codex authentication", async () => { let upstreamRequest; + let upstreamInit; globalThis.fetch = async (input, init) => { const request = input instanceof Request ? input : new Request(input, init); if (request.url.startsWith("https://control.test/")) { @@ -1060,6 +1062,7 @@ test("preserves a source Request while proxying Codex authentication", async () }); } upstreamRequest = request; + upstreamInit = init; return new Response(null, { status: 200 }); }; const loaded = await loadProxy("preserve-request"); @@ -1078,6 +1081,7 @@ test("preserves a source Request while proxying Codex authentication", async () assert.equal(upstreamRequest.headers.get("chatgpt-account-id"), "account-1"); assert.equal(upstreamRequest.headers.get("x-request-header"), "preserved"); assert.equal(await upstreamRequest.text(), "request-body"); + assert.equal(upstreamInit.timeout, false); }); test("preserves API-key requests if OpenAI authentication switches away from OAuth", async () => { diff --git a/packages/sandbox-runtime/tests/fixtures/reasoning-models.json b/packages/sandbox-runtime/tests/fixtures/reasoning-models.json index baa061f026..706458fcbf 100644 --- a/packages/sandbox-runtime/tests/fixtures/reasoning-models.json +++ b/packages/sandbox-runtime/tests/fixtures/reasoning-models.json @@ -132,6 +132,36 @@ "cache_write": 2.5 } }, + "claude-sonnet-5-5": { + "id": "claude-sonnet-5-5", + "name": "Claude Sonnet 5.5", + "family": "claude-sonnet", + "attachment": true, + "reasoning": true, + "reasoning_options": [ + { + "type": "effort", + "values": ["low", "medium", "high", "xhigh", "max"] + } + ], + "tool_call": true, + "temperature": false, + "release_date": "2026-09-28", + "modalities": { + "input": ["text", "image", "pdf"], + "output": ["text"] + }, + "limit": { + "context": 1000000, + "output": 128000 + }, + "cost": { + "input": 2, + "output": 10, + "cache_read": 0.2, + "cache_write": 2.5 + } + }, "claude-opus-4-5": { "id": "claude-opus-4-5", "name": "Claude Opus 4.5 (latest)", diff --git a/packages/sandbox-runtime/tests/test_claude_harness.py b/packages/sandbox-runtime/tests/test_claude_harness.py index ed0918791a..826f7708e4 100644 --- a/packages/sandbox-runtime/tests/test_claude_harness.py +++ b/packages/sandbox-runtime/tests/test_claude_harness.py @@ -325,14 +325,14 @@ async def test_options_follow_the_design_mapping(self, tmp_path: Path) -> None: HarnessPrompt( message_id="m1", text="hi", - model="anthropic/claude-opus-4-6", + model="anthropic/claude-sonnet-5-5", reasoning_effort="high", ), ) options = h.client.options assert options["cwd"] == str(tmp_path / "repo") assert options["cli_path"] == str(h.harness.wrapper_path) - assert options["model"] == "claude-opus-4-6" + assert options["model"] == "claude-sonnet-5-5" assert options["effort"] == "high" assert options["permission_mode"] == "dontAsk" assert options["disallowed_tools"] == ["AskUserQuestion"] diff --git a/packages/sandbox-runtime/tests/test_opencode_reasoning_contract.py b/packages/sandbox-runtime/tests/test_opencode_reasoning_contract.py index 7e12bdbb27..8fda795110 100644 --- a/packages/sandbox-runtime/tests/test_opencode_reasoning_contract.py +++ b/packages/sandbox-runtime/tests/test_opencode_reasoning_contract.py @@ -9,7 +9,9 @@ Source SHA-256: c9d3dc07540cf91a7b7362a3b42943132965f4ee6e8374113ee4f0f7b56b3d90 Claude Opus 5.5 added from the 2026-09-23 retrieval. Source SHA-256: e20acec396a73dc3db45d0eca7f0ede5bff28f09f002ba96ce7b1b566de7b6d0 -Subset SHA-256: e2b3a03acdfc3551f2e1752e26578768fa3265374b0fc79432392e4ce01d061d +Claude Sonnet 5.5 added from the 2026-09-28 retrieval. +Source SHA-256: 06e0071dd4ae9c9da2db1fabf28eb4994914fefdc5dd10270a5b340c88a49aec +Subset SHA-256: e9c9cc6f90fa9afbc75a2f18bf564398594d3f51693667cf1efd229617aaab0b Reconcile this frozen fixture with shared model/effort definitions when changing models or the binary. Mocks verify serialization, not live provider acceptance. """ diff --git a/packages/sandbox-runtime/uv.lock b/packages/sandbox-runtime/uv.lock index 064d54ce39..9ca7863f41 100644 --- a/packages/sandbox-runtime/uv.lock +++ b/packages/sandbox-runtime/uv.lock @@ -175,7 +175,7 @@ wheels = [ [[package]] name = "claude-agent-sdk" -version = "0.2.158" +version = "0.2.161" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, @@ -183,13 +183,12 @@ dependencies = [ { name = "mcp" }, { name = "sniffio" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/0e/5c/1d348e677fb199978d2b559fb5822fc58d77927f0988d1eb2acc18d5a0bb/claude_agent_sdk-0.2.158.tar.gz", hash = "sha256:672ddb19df334e9cb80bb264a6ddf00870c8d4e170b3fd0a3cf0ce354f634e7f", size = 354570, upload-time = "2026-09-23T01:39:14.77Z" } +sdist = { url = "https://files.pythonhosted.org/packages/8b/e1/8c94ff7f200eabcee9a9e57825dfac596bb6fbdeeafa594f49bc7e7f89b5/claude_agent_sdk-0.2.161.tar.gz", hash = "sha256:284a1372ec3af133578009ab2b821a238272dbf17d687a58be9df918d7b3ae89", size = 365168, upload-time = "2026-09-28T18:53:39.197Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/89/ad/37559b5a12170699eb6d4c5e1c66f491c4a3d7366f8f300a8dba0fc40aec/claude_agent_sdk-0.2.158-py3-none-macosx_11_0_arm64.whl", hash = "sha256:d124bd374dfbbcbdb4326e922f30d436251ae469df9cae545ef8ff6839d76e47", size = 90406616, upload-time = "2026-09-23T01:39:18.829Z" }, - { url = "https://files.pythonhosted.org/packages/c1/52/21968fc645ee206bb260f18b26ae8ae035fc5af820a7820b01e308496510/claude_agent_sdk-0.2.158-py3-none-macosx_11_0_x86_64.whl", hash = "sha256:beb963aa66eac715ff005e44ff02065fd5d855a10a47bc802097a31a522e6be3", size = 94751713, upload-time = "2026-09-23T01:39:22.591Z" }, - { url = "https://files.pythonhosted.org/packages/a1/4b/b42ab1a7ba32cea66184dedf8da158d49d00d6af753bd48998dab6e7a1d6/claude_agent_sdk-0.2.158-py3-none-manylinux_2_17_aarch64.whl", hash = "sha256:12f432273c3f0da103ea8dc921c3ce96e802c044e1f3f2535cf98b7d59777aa7", size = 99897552, upload-time = "2026-09-23T01:39:26.933Z" }, - { url = "https://files.pythonhosted.org/packages/e3/eb/1b34083c59a09a1a380c211eacd1cb185de4adc4bdb1170997e37f53f6ca/claude_agent_sdk-0.2.158-py3-none-manylinux_2_17_x86_64.whl", hash = "sha256:1f46ddac2cb51f16b2ca064a0dbeeab7db712f34e72454cc7776713e0f0f663c", size = 100257803, upload-time = "2026-09-23T01:39:31.501Z" }, - { url = "https://files.pythonhosted.org/packages/15/0d/11ab53934b04176180099a79bb186036e304e1fe1310bdecf21df599f1d3/claude_agent_sdk-0.2.158-py3-none-win_amd64.whl", hash = "sha256:b1e7f6b9d048c2c44e5228d6a0eeb19ae3aafb643c1115c456ea9937e0439aef", size = 102853711, upload-time = "2026-09-23T01:39:36.133Z" }, + { url = "https://files.pythonhosted.org/packages/e8/be/528e3ec2bb30deff4fcc9484dd47d1022786ad3659d31110bd0ea66ea548/claude_agent_sdk-0.2.161-py3-none-macosx_11_0_arm64.whl", hash = "sha256:127c18a790dc9f77cc2d648f932e11b06cd3325c4b3de7a73d3bcf3d3b74b8bc", size = 93872213, upload-time = "2026-09-28T18:53:42.956Z" }, + { url = "https://files.pythonhosted.org/packages/17/1c/a0dcb628ac2f25bc92d858857a399202723098057828a909ca4f9893b721/claude_agent_sdk-0.2.161-py3-none-macosx_11_0_x86_64.whl", hash = "sha256:599e1c8ccb4d48ed57999b48e80937d6a6aefbc52b0c0cda1291aea21df29e3e", size = 98316495, upload-time = "2026-09-28T18:53:47.239Z" }, + { url = "https://files.pythonhosted.org/packages/73/04/3c0caee46c27eee75642b52ee11857c0d3b9d6917ecfd916da2ffc2c2d3f/claude_agent_sdk-0.2.161-py3-none-manylinux_2_17_aarch64.whl", hash = "sha256:18081d3277c07a71891bd15d1496387d9efc5f04a6b482ae3a03e7a0b8067069", size = 103331951, upload-time = "2026-09-28T18:53:51.367Z" }, + { url = "https://files.pythonhosted.org/packages/09/70/2c7c4160a19c6edd42e317bdc9cf810e91b6a535c04377fb09d4ee090769/claude_agent_sdk-0.2.161-py3-none-manylinux_2_17_x86_64.whl", hash = "sha256:b854e46d2889a51e1a77978711a05c757e63064bd5d8f89bcf74bb507babfc2c", size = 103684983, upload-time = "2026-09-28T18:53:55.496Z" }, ] [[package]] @@ -558,7 +557,7 @@ dev = [ [package.metadata] requires-dist = [ - { name = "claude-agent-sdk", specifier = "==0.2.158" }, + { name = "claude-agent-sdk", specifier = "==0.2.161" }, { name = "cryptography", specifier = ">=44.0.0" }, { name = "httpx", specifier = ">=0.27.0" }, { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.14.0" }, diff --git a/packages/shared/src/models.test.ts b/packages/shared/src/models.test.ts index 3697f7ecef..390bd3ef45 100644 --- a/packages/shared/src/models.test.ts +++ b/packages/shared/src/models.test.ts @@ -1,3 +1,4 @@ +import { readFileSync } from "node:fs"; import { describe, expect, it } from "vitest"; import { DEFAULT_ENABLED_MODELS, @@ -25,6 +26,7 @@ const ANTHROPIC_MODELS = [ "anthropic/claude-sonnet-4-5", "anthropic/claude-sonnet-4-6", "anthropic/claude-sonnet-5", + "anthropic/claude-sonnet-5-5", "anthropic/claude-opus-4-5", "anthropic/claude-opus-4-6", "anthropic/claude-opus-4-7", @@ -92,6 +94,37 @@ const GO_MODELS = [ const DEEPSEEK_MODELS = ["deepseek/deepseek-v4-flash", "deepseek/deepseek-v4-pro"] as const; const ZAI_CODING_PLAN_MODELS = ["zai-coding-plan/glm-5.2", "zai-coding-plan/glm-5.3"] as const; +it.each([ + "../../../docs/AVAILABLE_MODELS.md", + "../../../packages/docs/content/docs/models/choosing-a-model.mdx", +])("keeps the model inventory in %s aligned with the catalog", (file) => { + const rows = readFileSync(new URL(file, import.meta.url), "utf8") + .split("\n") + .filter((line) => /^\| `[^`]+`/.test(line)) + .map((line) => + line + .split("|") + .slice(1, -1) + .map((cell) => cell.trim()) + ); + const models = MODEL_CATALOG.flatMap((group) => group.models); + + expect(rows.map(([id]) => id).sort()).toEqual(models.map(({ id }) => `\`${id}\``).sort()); + for (const model of models) { + const row = rows.find(([id]) => id === `\`${model.id}\``); + expect(row?.[1]).toBe(model.name); + if (file.endsWith("AVAILABLE_MODELS.md")) { + expect(row?.[2]).toBe(model.description); + } + if ("reasoning" in model) { + expect(row?.at(-2)).toBe(model.reasoning.efforts.join(", ")); + expect(row?.at(-1)).toBe(model.reasoning.default ?? "Not set"); + } else { + expect(row?.at(-1)).toBe("N/A"); + } + } +}); + describe("model utilities", () => { it("derives every public model view from the authoritative catalog", () => { const catalogModels = MODEL_CATALOG.flatMap((group) => group.models); @@ -151,6 +184,7 @@ describe("model utilities", () => { expect(normalizeModelId("claude-sonnet-4-6")).toBe("anthropic/claude-sonnet-4-6"); expect(normalizeModelId("claude-opus-4-8")).toBe("anthropic/claude-opus-4-8"); expect(normalizeModelId("claude-opus-5")).toBe("anthropic/claude-opus-5"); + expect(normalizeModelId("claude-sonnet-5-5")).toBe("anthropic/claude-sonnet-5-5"); expect(normalizeModelId("claude-fable-5")).toBe("anthropic/claude-fable-5"); expect(normalizeModelId("claude-fable-5-1")).toBe("anthropic/claude-fable-5-1"); expect(normalizeModelId("gpt-6-sol")).toBe("openai/gpt-6-sol"); @@ -158,6 +192,7 @@ describe("model utilities", () => { expect(isValidModel("claude-sonnet-4-6")).toBe(true); expect(isValidModel("claude-opus-4-8")).toBe(true); expect(isValidModel("claude-opus-5")).toBe(true); + expect(isValidModel("claude-sonnet-5-5")).toBe(true); expect(isValidModel("claude-fable-5")).toBe(true); expect(isValidModel("claude-fable-5-1")).toBe(true); expect(isValidModel("gpt-6-sol")).toBe(true); @@ -355,6 +390,7 @@ describe("model utilities", () => { expect(getDefaultReasoningEffort("anthropic/claude-sonnet-4-6")).toBe("high"); expect(getDefaultReasoningEffort("anthropic/claude-opus-4-8")).toBe("high"); expect(getDefaultReasoningEffort("anthropic/claude-sonnet-5")).toBe("high"); + expect(getDefaultReasoningEffort("anthropic/claude-sonnet-5-5")).toBe("high"); expect(getDefaultReasoningEffort("anthropic/claude-opus-5")).toBe("high"); expect(getDefaultReasoningEffort("anthropic/claude-opus-5-5")).toBe("high"); expect(getDefaultReasoningEffort("anthropic/claude-fable-5")).toBe("high"); @@ -381,6 +417,10 @@ describe("model utilities", () => { efforts: ["low", "medium", "high", "xhigh", "max"], default: "high", }); + expect(getReasoningConfig("anthropic/claude-sonnet-5-5")).toEqual({ + efforts: ["low", "medium", "high", "xhigh", "max"], + default: "high", + }); expect(getReasoningConfig("anthropic/claude-opus-4-8")).toEqual({ efforts: ["low", "medium", "high", "xhigh", "max"], default: "high", @@ -448,6 +488,8 @@ describe("model utilities", () => { expect(isValidReasoningEffort("anthropic/claude-opus-4-8", "xhigh")).toBe(true); expect(isValidReasoningEffort("anthropic/claude-opus-4-8", "none")).toBe(false); expect(isValidReasoningEffort("anthropic/claude-sonnet-5", "xhigh")).toBe(true); + expect(isValidReasoningEffort("anthropic/claude-sonnet-5-5", "max")).toBe(true); + expect(isValidReasoningEffort("anthropic/claude-sonnet-5-5", "none")).toBe(false); expect(isValidReasoningEffort("anthropic/claude-opus-5", "xhigh")).toBe(true); expect(isValidReasoningEffort("anthropic/claude-opus-5", "none")).toBe(false); expect(isValidReasoningEffort("anthropic/claude-opus-5-5", "xhigh")).toBe(true); diff --git a/packages/shared/src/models.ts b/packages/shared/src/models.ts index 385f5980ed..d06e76b856 100644 --- a/packages/shared/src/models.ts +++ b/packages/shared/src/models.ts @@ -70,7 +70,16 @@ export const MODEL_CATALOG = [ { id: "anthropic/claude-sonnet-5", name: "Claude Sonnet 5", - description: "Latest Sonnet, adaptive thinking", + description: "Balanced performance, adaptive thinking", + reasoning: { + efforts: ["low", "medium", "high", "xhigh", "max"], + default: "high", + }, + }, + { + id: "anthropic/claude-sonnet-5-5", + name: "Claude Sonnet 5.5", + description: "Latest Sonnet, fast and intelligent", reasoning: { efforts: ["low", "medium", "high", "xhigh", "max"], default: "high", diff --git a/packages/shared/src/types/audit-events.test.ts b/packages/shared/src/types/audit-events.test.ts index e55945a695..9f1ca0c704 100644 --- a/packages/shared/src/types/audit-events.test.ts +++ b/packages/shared/src/types/audit-events.test.ts @@ -124,6 +124,7 @@ describe("interpretAuditEvent", () => { }); it.each([ + "session.private_break_glass", "team.created", "team.updated", "team.archived", diff --git a/packages/shared/src/types/audit-events.ts b/packages/shared/src/types/audit-events.ts index 1c52d13b1d..5e313f4afa 100644 --- a/packages/shared/src/types/audit-events.ts +++ b/packages/shared/src/types/audit-events.ts @@ -71,6 +71,7 @@ export const AUTHORIZATION_DECISION_ACTIONS = { /** Actions written by the operation owner alongside the change; their result is the domain outcome. */ export const AUDIT_OPERATION_ACTIONS = [ + "session.private_break_glass", "workspace.member_role_updated", "workspace.member_status_updated", "workspace.default_role_assigned", diff --git a/packages/shared/src/types/session-archive.ts b/packages/shared/src/types/session-archive.ts index c2de608030..a368126757 100644 --- a/packages/shared/src/types/session-archive.ts +++ b/packages/shared/src/types/session-archive.ts @@ -42,6 +42,12 @@ export type SessionBatchArchiveResult = z.infer; diff --git a/packages/web/src/components/session-header.test.tsx b/packages/web/src/components/session-header.test.tsx index 12bfb0f2fb..136368b023 100644 --- a/packages/web/src/components/session-header.test.tsx +++ b/packages/web/src/components/session-header.test.tsx @@ -87,6 +87,39 @@ function member(repoOwner: string, repoName: string, position: number) { } describe("SessionHeader", () => { + it("gives the desktop title available header space instead of a fixed width", () => { + const title = "Correctness review of background agents"; + render( + ()} + actionsButtonRef={createRef()} + onToggleDetails={vi.fn()} + onToggleDesktopDetails={vi.fn()} + onOpenMobileDetails={vi.fn()} + actions={actions} + renameSession={vi.fn()} + /> + ); + + const heading = screen.getByRole("heading", { level: 1 }); + expect(heading).not.toHaveClass("md:max-w-40"); + expect(heading.parentElement).not.toHaveClass("md:flex-initial"); + expect(heading.parentElement?.parentElement).not.toHaveClass("md:flex-initial"); + expect(screen.getByText("acme/web")).toHaveClass("truncate"); + + fireEvent.click(screen.getByRole("button", { name: title })); + const input = screen.getByRole("textbox", { name: "Session title" }); + expect(input).toHaveClass("w-full"); + expect(input).not.toHaveClass("md:max-w-40"); + }); + it("disables lifecycle controls and connection UI for a read-only session", async () => { render( -
-
+
+
{!isOpen && } -
+
{isRenaming ? ( ) : ( -

+

)} -

{repoLabel}

+

{repoLabel}

diff --git a/packages/web/src/components/session-right-sidebar.test.tsx b/packages/web/src/components/session-right-sidebar.test.tsx index 6e3afc673b..eb40ca570c 100644 --- a/packages/web/src/components/session-right-sidebar.test.tsx +++ b/packages/web/src/components/session-right-sidebar.test.tsx @@ -132,6 +132,35 @@ describe("SessionRightSidebar", () => { expect(click.mock.instances[0]).toHaveProperty("href", "blob:session-trace"); }); + it.each(["session_error", "error"])( + "reports a %s export record as a failed download", + async (type) => { + vi.mocked(browserApiFetch).mockResolvedValueOnce( + new Response(`{"schemaVersion":1,"type":"${type}"}\n`, { + headers: { "Content-Type": "application/x-ndjson" }, + }) + ); + const click = vi.spyOn(HTMLAnchorElement.prototype, "click").mockImplementation(() => {}); + render( + + ); + + fireEvent.click(screen.getByRole("button", { name: "Download trace" })); + await waitFor(() => expect(toast.error).toHaveBeenCalledWith("Failed to download trace")); + expect(click).not.toHaveBeenCalled(); + expect(URL.createObjectURL).not.toHaveBeenCalled(); + } + ); + it("aborts a trace body that stalls after headers and re-enables the download button", async () => { vi.useFakeTimers(); let fetchSignal: AbortSignal | undefined; diff --git a/packages/web/src/components/session-right-sidebar.tsx b/packages/web/src/components/session-right-sidebar.tsx index 79201f3bd7..3507285e69 100644 --- a/packages/web/src/components/session-right-sidebar.tsx +++ b/packages/web/src/components/session-right-sidebar.tsx @@ -110,7 +110,16 @@ export function SessionRightSidebarContent({ ); if (!response.ok) throw new Error("Trace export failed"); - const url = URL.createObjectURL(await response.blob()); + const blob = await response.blob(); + // Trace read and stream failures arrive as NDJSON records inside a 200 response. + const failed = (await blob.text()).split("\n").some((line) => { + if (!line) return false; + const { type } = JSON.parse(line) as { type?: unknown }; + return type === "session_error" || type === "error"; + }); + if (failed) throw new Error("Trace export failed"); + + const url = URL.createObjectURL(blob); const link = document.createElement("a"); link.href = url; link.download = `session-${sessionId}.ndjson`; diff --git a/packages/web/src/components/settings/audit-log-settings.test.tsx b/packages/web/src/components/settings/audit-log-settings.test.tsx index 5335585ab9..0e51cb3b5d 100644 --- a/packages/web/src/components/settings/audit-log-settings.test.tsx +++ b/packages/web/src/components/settings/audit-log-settings.test.tsx @@ -110,6 +110,12 @@ describe("AuditLogSettings", () => { expect(article.getByText("Applied")).toBeInTheDocument(); }); + it("labels private session break-glass reads as operations", () => { + const article = renderSingle(createEvent("applied", { action: "session.private_break_glass" })); + expect(article.getByText("Private session break-glass read")).toBeInTheDocument(); + expect(article.getByText("Applied")).toBeInTheDocument(); + }); + it("renders outcomes, stable summaries, timestamps, and expandable structured details", async () => { hook.events = [ createEvent("applied"), diff --git a/packages/web/src/components/settings/audit-log-settings.tsx b/packages/web/src/components/settings/audit-log-settings.tsx index bb5971dfbd..470baeea70 100644 --- a/packages/web/src/components/settings/audit-log-settings.tsx +++ b/packages/web/src/components/settings/audit-log-settings.tsx @@ -41,6 +41,7 @@ const UNRECOGNIZED: BadgeTreatment = { }; const OPERATION_LABELS: Record = { + "session.private_break_glass": "Private session break-glass read", "workspace.member_role_updated": "Member role updated", "workspace.member_status_updated": "Member status updated", "workspace.default_role_assigned": "Default role assigned", diff --git a/terraform/environments/aws-production/main.tf b/terraform/environments/aws-production/main.tf index ff198baa84..26868c6205 100644 --- a/terraform/environments/aws-production/main.tf +++ b/terraform/environments/aws-production/main.tf @@ -44,6 +44,7 @@ module "control_plane" { LOG_LEVEL = "info" SANDBOX_PROVIDER = "modal" UNSAFE_ALLOW_ALL_USERS = "false" + TEAMS_ENFORCEMENT = "shadow" }, var.config) tags = { Environment = local.environment } diff --git a/terraform/environments/production/variables.tf b/terraform/environments/production/variables.tf index 7d2f479465..b77fb2c8e2 100644 --- a/terraform/environments/production/variables.tf +++ b/terraform/environments/production/variables.tf @@ -745,6 +745,17 @@ variable "sandbox_inactivity_timeout_ms" { default = 600000 } +variable "teams_enforcement" { + description = "Session team authorization mode; private visibility applies in every mode." + type = string + default = "shadow" + + validation { + condition = contains(["off", "shadow", "on"], var.teams_enforcement) + error_message = "teams_enforcement must be 'off', 'shadow', or 'on'." + } +} + variable "sandbox_boot_timeout_ms" { description = "Milliseconds a sandbox whose bridge has connected may keep booting (clone, setup.sh, start.sh, agent start) before OpenInspect fails it and the prompt it was for." type = number diff --git a/terraform/environments/production/workers-control-plane.tf b/terraform/environments/production/workers-control-plane.tf index 3181635d40..b3ede005d1 100644 --- a/terraform/environments/production/workers-control-plane.tf +++ b/terraform/environments/production/workers-control-plane.tf @@ -109,6 +109,7 @@ module "control_plane_worker" { SANDBOX_PROVIDER = { value = var.sandbox_provider } SANDBOX_INACTIVITY_TIMEOUT_MS = { value = tostring(var.sandbox_inactivity_timeout_ms) } SANDBOX_BOOT_TIMEOUT_MS = { value = tostring(var.sandbox_boot_timeout_ms) } + TEAMS_ENFORCEMENT = { value = var.teams_enforcement } }, local.github_oauth_enabled ? { GITHUB_CLIENT_ID = { value = trimspace(var.github_client_id) }