From a5cfc253c73691847544a123ecaf96c9e780ac32 Mon Sep 17 00:00:00 2001 From: Cole Murray Date: Mon, 28 Sep 2026 09:25:34 -0700 Subject: [PATCH 01/13] fix(web): prevent desktop session titles from truncating prematurely (#2117) ## Summary - Remove the fixed 160px desktop width from session titles and rename inputs so they use available header space. - Keep the title and repository label responsive when the window is narrow. - Add a regression test for the desktop header layout and rename field. ## Verification - `npm test -w @open-inspect/web` (226 files, 1,980 tests passed) - `npm run lint -w @open-inspect/web` - `npm run typecheck -w @open-inspect/web` - `git diff --check` --- *Created with [Open-Inspect](https://open-inspect-prod.vercel.app/session/9fabc27d2a3552ca949adecac0c09e55)* ## Summary by CodeRabbit * **Improvements** * Session titles and repository labels can use more available horizontal space on medium and larger screens. * Repository labels now truncate when space is limited. * Header items have more spacing at medium screen widths. --------- Co-authored-by: Cole Murray <2492022+ColeMurray@users.noreply.github.com> Co-authored-by: waclaude --- .../src/components/session-header.test.tsx | 33 +++++++++++++++++++ .../web/src/components/session-header.tsx | 12 +++---- 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/packages/web/src/components/session-header.test.tsx b/packages/web/src/components/session-header.test.tsx index 12bfb0f2fb..136368b023 100644 --- a/packages/web/src/components/session-header.test.tsx +++ b/packages/web/src/components/session-header.test.tsx @@ -87,6 +87,39 @@ function member(repoOwner: string, repoName: string, position: number) { } describe("SessionHeader", () => { + it("gives the desktop title available header space instead of a fixed width", () => { + const title = "Correctness review of background agents"; + render( + ()} + actionsButtonRef={createRef()} + onToggleDetails={vi.fn()} + onToggleDesktopDetails={vi.fn()} + onOpenMobileDetails={vi.fn()} + actions={actions} + renameSession={vi.fn()} + /> + ); + + const heading = screen.getByRole("heading", { level: 1 }); + expect(heading).not.toHaveClass("md:max-w-40"); + expect(heading.parentElement).not.toHaveClass("md:flex-initial"); + expect(heading.parentElement?.parentElement).not.toHaveClass("md:flex-initial"); + expect(screen.getByText("acme/web")).toHaveClass("truncate"); + + fireEvent.click(screen.getByRole("button", { name: title })); + const input = screen.getByRole("textbox", { name: "Session title" }); + expect(input).toHaveClass("w-full"); + expect(input).not.toHaveClass("md:max-w-40"); + }); + it("disables lifecycle controls and connection UI for a read-only session", async () => { render( -
-
+
+
{!isOpen && } -
+
{isRenaming ? ( ) : ( -

+

)} -

{repoLabel}

+

{repoLabel}

From 1490806d32e6189ab0a32e86962073d5f05dbb0e Mon Sep 17 00:00:00 2001 From: Cole Murray Date: Mon, 28 Sep 2026 19:17:56 -0700 Subject: [PATCH 02/13] refactor: split Modal sandbox manager responsibilities (#2015) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - Reduce `manager.py` from 722 to 294 lines (59%) by extracting concrete `SandboxLauncher` and `SandboxTunnels` collaborators, plus shared configuration/handle records. - Compute service-port ownership once for encrypted ports, runtime environment, URL routing, and best-effort tunnel publication. - Preserve public manager imports/signatures, image selection/error classification, environment precedence, legacy restore credentials, snapshot deadlines, and termination behavior. - Review follow-up: explicitly reject Boolean extra tunnel ports, restore all 17 legacy manager constant exports, and name retry defaults. ## Design | Component | Responsibility | | --- | --- | | `SandboxManager` | Create/restore normalization and logging; snapshot, lookup, termination | | `SandboxLauncher` | Image resolution, environment/credentials, resources, Modal create, handle assembly | | `SandboxTunnels` | Port ownership, runtime port settings, URL resolution/retries, tunnel-file publication | | `models.py` | Shared launch configuration and returned handle | Launch execution flows from manager to launcher to tunnels; compatibility exports reference the canonical constants. Collaborators never import the manager. No generic interfaces, provider registry, new lifecycle authority, or private forwarding wrappers. Across the four production files, total size increases by 65 lines for explicit module boundaries, exports, and named tunnel results. ## Verification - Baseline: 257 Modal tests passed before changes. - Python 3.12: `uv run --extra dev pytest tests/ -q` — **299 passed**. - `uv run --extra dev ruff check src/ tests/` — passed. - `uv run --extra dev ruff format --check src/ tests/` — passed. - `git diff --check` — passed. - `uv run --extra dev mypy src/`: unchanged base has 20 diagnostics; this branch has 15, all in unchanged files (`clone_token.py`, `build_session.py`, `web_api.py`). No diagnostics in the changed/new modules. - Launch matrix exercises real manager/launcher/tunnel composition with Modal I/O mocked. Added coverage for missing versus transient image lookup/spawn errors, absence of fallback/retry, partial/exhausted tunnel resolution, retry delays, and non-fatal file-write failures. - Added regressions for all 17 legacy manager constants and six create/restore cases rejecting Boolean tunnel ports without losing valid ports or consuming the port limit. No live Modal deployment or billable provider canary was performed. ## Summary by CodeRabbit * **New Features** * Sandboxes can be launched from base images, repository images, or snapshots. * Launch results include available service connection URLs, credentials, and snapshot details. * Configure service and user tunnel ports for code-server, VNC, terminal access, and other services. * Tunnel URLs are resolved and made available through the sandbox environment. * Invalid and duplicate tunnel port settings are filtered out. * Existing sandbox allocations can be reused when their ownership matches. * **Bug Fixes** * Missing repository images are reported clearly. * Sandboxes remain available when tunnel resolution or environment-file updates fail. --------- Co-authored-by: Cole Murray <2492022+ColeMurray@users.noreply.github.com> --- packages/modal-infra/src/sandbox/launch.py | 354 +++++++++ packages/modal-infra/src/sandbox/manager.py | 671 ++---------------- packages/modal-infra/src/sandbox/models.py | 59 ++ packages/modal-infra/src/sandbox/tunnels.py | 215 ++++++ .../tests/test_agent_slack_notify_env.py | 11 +- .../modal-infra/tests/test_code_server.py | 62 +- .../modal-infra/tests/test_llm_secrets.py | 4 +- .../modal-infra/tests/test_manager_exports.py | 45 ++ .../tests/test_sandbox_env_vars.py | 59 +- .../modal-infra/tests/test_sandbox_launch.py | 315 +++++--- .../tests/test_sandbox_resources.py | 19 +- packages/modal-infra/tests/test_ttyd.py | 168 ++--- .../modal-infra/tests/test_tunnel_ports.py | 436 +++++------- packages/modal-infra/tests/test_vnc.py | 65 +- 14 files changed, 1328 insertions(+), 1155 deletions(-) create mode 100644 packages/modal-infra/src/sandbox/launch.py create mode 100644 packages/modal-infra/src/sandbox/models.py create mode 100644 packages/modal-infra/src/sandbox/tunnels.py create mode 100644 packages/modal-infra/tests/test_manager_exports.py diff --git a/packages/modal-infra/src/sandbox/launch.py b/packages/modal-infra/src/sandbox/launch.py new file mode 100644 index 0000000000..7aee1fbb04 --- /dev/null +++ b/packages/modal-infra/src/sandbox/launch.py @@ -0,0 +1,354 @@ +"""Translate a session launch into Modal image, environment, and resource arguments.""" + +import json +import secrets +import time +from dataclasses import dataclass +from typing import Any + +import modal + +from sandbox_runtime.constants import ( + DOCKER_ENABLED_ENV_VAR, + NOVNC_PORT_ENV_VAR, + SANDBOX_TIMEOUT_ENV_VAR, + VNC_PASSWORD_ENV_VAR, + VNC_PASSWORD_MAX_BYTES, +) +from sandbox_runtime.log_config import get_logger +from sandbox_runtime.types import SandboxStatus + +from ..app import app +from ..app_config import APP_NAME +from ..images.base import base_image +from .launch_policy import ( + docker_allocation_name, + docker_allocation_tags, + docker_base_image, + docker_runtime_env, + launch_kwargs, + parse_launch, +) +from .models import SandboxConfig, SandboxHandle +from .tunnels import SandboxTunnels +from .vcs_env import inject_vcs_env_vars + +_RESERVED_LAUNCH_ENV_VARS = { + "RESTORED_FROM_SNAPSHOT", + "FROM_REPO_IMAGE", + "REPO_IMAGE_SHA", + "IMAGE_BUILD_MODE", + "TERMINAL_ENABLED", + "AGENT_SLACK_NOTIFY_ENABLED", + "SESSION_CONFIG", + VNC_PASSWORD_ENV_VAR, + NOVNC_PORT_ENV_VAR, + DOCKER_ENABLED_ENV_VAR, +} + +log = get_logger("manager") +ACCESS_PASSWORD_READ_TIMEOUT_SECONDS = 30 + + +class RepositoryImageUnavailableError(RuntimeError): + """The selected repository image no longer exists in Modal.""" + + +@dataclass(frozen=True) +class BaseImageSource: + pass + + +@dataclass(frozen=True) +class RepositoryImageSource: + image_id: str + sha: str | None + + +@dataclass(frozen=True) +class SnapshotImageSource: + image_id: str + clone_token: str | None + + +type SandboxImageSource = BaseImageSource | RepositoryImageSource | SnapshotImageSource + + +async def _create_sandbox( + create_kwargs: dict[str, Any], *, repository_image: bool +) -> modal.Sandbox: + """Only a missing repository image at create time is classified as unavailable.""" + try: + return await modal.Sandbox.create.aio( + "python", "-m", "sandbox_runtime.entrypoint", **create_kwargs + ) + except modal.exception.NotFoundError as e: + if repository_image: + raise RepositoryImageUnavailableError("repository image is unavailable") from e + raise + + +def _session_identity(session_config: Any) -> str: + if isinstance(session_config, dict): + session_id = session_config.get("session_id") + elif session_config is not None: + session_id = session_config.session_id + else: + session_id = None + return session_id if isinstance(session_id, str) else "" + + +@dataclass(frozen=True) +class SandboxLaunchSpec: + """Canonical launch configuration paired with one image source variant.""" + + config: SandboxConfig + source: SandboxImageSource + + +class SandboxLauncher: + """Own the common Modal launch path for base, repository, and snapshot images.""" + + @staticmethod + def _generate_code_server_password() -> str: + """Generate a random code-server password.""" + return secrets.token_urlsafe(16) + + @staticmethod + def _generate_vnc_password() -> str: + """Generate a random VNC password.""" + return secrets.token_urlsafe(VNC_PASSWORD_MAX_BYTES)[:VNC_PASSWORD_MAX_BYTES] + + async def launch(self, spec: SandboxLaunchSpec) -> SandboxHandle: + """Launch a Modal sandbox from a normalized create or restore specification.""" + config = spec.config + has_repository = bool(config.repo_owner) + sandbox_id = config.sandbox_id + if not sandbox_id: + sandbox_name = ( + f"{config.repo_owner}-{config.repo_name}" if has_repository else "no-repository" + ) + sandbox_id = f"sandbox-{sandbox_name}-{int(time.time() * 1000)}" + + docker = parse_launch(config.sandbox_backend, config.settings) + env_vars = { + key: value + for key, value in (config.user_env_vars or {}).items() + if key not in _RESERVED_LAUNCH_ENV_VARS + } + env_vars.update( + { + "PYTHONUNBUFFERED": "1", + "SANDBOX_ID": sandbox_id, + "CONTROL_PLANE_URL": config.control_plane_url, + "SANDBOX_AUTH_TOKEN": config.sandbox_auth_token, + SANDBOX_TIMEOUT_ENV_VAR: str(config.timeout_seconds), + "REPO_OWNER": config.repo_owner or "", + "REPO_NAME": config.repo_name or "", + **docker_runtime_env(docker), + } + ) + + clone_token: str | None = None + include_github_cli_aliases = False + snapshot_id: str | None = None + if isinstance(spec.source, BaseImageSource): + image = docker_base_image() if docker.enabled else base_image + elif isinstance(spec.source, RepositoryImageSource): + try: + image = modal.Image.from_id(spec.source.image_id) + except modal.exception.NotFoundError as e: + raise RepositoryImageUnavailableError("repository image is unavailable") from e + env_vars["FROM_REPO_IMAGE"] = "true" + env_vars["REPO_IMAGE_SHA"] = spec.source.sha or "" + else: + image = modal.Image.from_id(spec.source.image_id) + env_vars["RESTORED_FROM_SNAPSHOT"] = "true" + clone_token = spec.source.clone_token + include_github_cli_aliases = True + snapshot_id = spec.source.image_id + + if config.session_config is not None: + env_vars["SESSION_CONFIG"] = ( + json.dumps(config.session_config) + if isinstance(config.session_config, dict) + else config.session_config.model_dump_json() + ) + + inject_vcs_env_vars( + env_vars, + clone_token=clone_token if has_repository else None, + include_github_cli_aliases=include_github_cli_aliases, + ) + + code_server_password: str | None = None + if config.code_server_enabled: + code_server_password = self._generate_code_server_password() + env_vars["CODE_SERVER_PASSWORD"] = code_server_password + + vnc_password: str | None = None + if config.vnc_enabled: + vnc_password = self._generate_vnc_password() + env_vars[VNC_PASSWORD_ENV_VAR] = vnc_password + + if config.agent_slack_notify_enabled: + env_vars["AGENT_SLACK_NOTIFY_ENABLED"] = "true" + + tunnels = SandboxTunnels( + code_server_enabled=config.code_server_enabled, + vnc_enabled=config.vnc_enabled, + settings=config.settings, + ) + env_vars.update(tunnels.environment) + + # A fresh handle avoids Modal caching the ID of a deleted/recreated secret. + llm_secrets = modal.Secret.from_name("llm-api-keys") + await llm_secrets.hydrate.aio() + create_kwargs: dict[str, Any] = { + "image": image, + "app": app, + "secrets": [llm_secrets], + "timeout": config.timeout_seconds, + "workdir": "/workspace", + "env": env_vars, + **launch_kwargs(docker), + } + if tunnels.exposed_ports: + create_kwargs["encrypted_ports"] = tunnels.exposed_ports + + repository_image = isinstance(spec.source, RepositoryImageSource) + if docker.enabled: + sandbox, adopted = await self._launch_docker_sandbox( + session_id=_session_identity(config.session_config), + sandbox_id=sandbox_id, + retire_sandbox_id=config.retire_sandbox_id, + create_kwargs=create_kwargs, + repository_image=repository_image, + launch_deadline_at_ms=config.launch_deadline_at_ms, + ) + if adopted: + passwords = await self._read_access_passwords( + sandbox, + code_server_enabled=config.code_server_enabled, + vnc_enabled=config.vnc_enabled, + ) + code_server_password = passwords.get("CODE_SERVER_PASSWORD") + vnc_password = passwords.get(VNC_PASSWORD_ENV_VAR) + else: + sandbox = await _create_sandbox(create_kwargs, repository_image=repository_image) + modal_object_id = sandbox.object_id + urls = await tunnels.resolve(sandbox, sandbox_id) + + return SandboxHandle( + sandbox_id=sandbox_id, + modal_sandbox=sandbox, + status=SandboxStatus.WARMING, + created_at=time.time(), + snapshot_id=snapshot_id, + modal_object_id=modal_object_id, + code_server_url=urls.code_server_url, + code_server_password=code_server_password, + vnc_url=urls.vnc_url, + vnc_password=vnc_password, + ttyd_url=urls.ttyd_url, + tunnel_urls=urls.tunnel_urls, + sandbox_backend=docker.backend, + ) + + async def _launch_docker_sandbox( + self, + *, + session_id: str, + sandbox_id: str, + retire_sandbox_id: str | None, + create_kwargs: dict[str, Any], + repository_image: bool, + launch_deadline_at_ms: int | None = None, + ) -> tuple[modal.Sandbox, bool]: + """Create a named VM or adopt only the allocation owned by this generation.""" + if retire_sandbox_id: + await self._retire_docker_allocation(session_id, retire_sandbox_id) + name = docker_allocation_name(session_id) + tags = docker_allocation_tags(session_id, sandbox_id) + existing = await self._find_owned_docker_allocation(name, tags) + if existing is None: + if launch_deadline_at_ms is not None and time.time() * 1000 >= launch_deadline_at_ms: + raise RuntimeError("VM launch deadline expired") + try: + sandbox = await _create_sandbox( + {**create_kwargs, "name": name, "tags": tags}, + repository_image=repository_image, + ) + return sandbox, False + except modal.exception.AlreadyExistsError: + existing = await self._find_owned_docker_allocation(name, tags) + if existing is None: + raise + log.info( + "sandbox.docker_allocation_adopted", + sandbox_id=sandbox_id, + modal_object_id=existing.object_id, + ) + return existing, True + + @staticmethod + async def _read_access_passwords( + sandbox: modal.Sandbox, *, code_server_enabled: bool, vnc_enabled: bool + ) -> dict[str, str]: + """Recover enabled service credentials from the owned VM launch environment.""" + keys = [] + if code_server_enabled: + keys.append("CODE_SERVER_PASSWORD") + if vnc_enabled: + keys.append(VNC_PASSWORD_ENV_VAR) + if not keys: + return {} + process = await sandbox.exec.aio( + "python", + "-I", + "-c", + "import json, os, sys; print(json.dumps({k: os.environ.get(k) for k in sys.argv[1:]}))", + *keys, + timeout=ACCESS_PASSWORD_READ_TIMEOUT_SECONDS, + ) + output = await process.stdout.read.aio() + if await process.wait.aio() != 0: + raise RuntimeError("Could not recover adopted sandbox access credentials") + try: + passwords = json.loads(output) + except ValueError: + raise RuntimeError("Could not recover adopted sandbox access credentials") from None + if not isinstance(passwords, dict) or any( + not isinstance(passwords.get(key), str) or not passwords[key] for key in keys + ): + raise RuntimeError("Could not recover adopted sandbox access credentials") + return {key: passwords[key] for key in keys} + + @staticmethod + async def _find_owned_docker_allocation( + name: str, tags: dict[str, str] + ) -> modal.Sandbox | None: + try: + sandbox = await modal.Sandbox.from_name.aio(APP_NAME, name) + except modal.exception.NotFoundError: + return None + if await sandbox.get_tags.aio() != tags: + raise RuntimeError("Docker sandbox allocation ownership mismatch") + return sandbox + + async def _retire_docker_allocation(self, session_id: str, sandbox_id: str) -> None: + """Terminate a prior named VM only when its ownership tags match.""" + name = docker_allocation_name(session_id) + try: + sandbox = await modal.Sandbox.from_name.aio(APP_NAME, name) + except modal.exception.NotFoundError: + return + if await sandbox.get_tags.aio() != docker_allocation_tags(session_id, sandbox_id): + log.warn("sandbox.docker_allocation_retire_mismatch", sandbox_id=sandbox_id) + return + await sandbox.terminate.aio(wait=True) + log.info( + "sandbox.docker_allocation_retired", + sandbox_id=sandbox_id, + modal_object_id=sandbox.object_id, + ) diff --git a/packages/modal-infra/src/sandbox/manager.py b/packages/modal-infra/src/sandbox/manager.py index 30e044eb8e..7a82b3f9f8 100644 --- a/packages/modal-infra/src/sandbox/manager.py +++ b/packages/modal-infra/src/sandbox/manager.py @@ -1,18 +1,6 @@ -""" -Sandbox lifecycle management for Open-Inspect. +"""Provider lifecycle operations for Open-Inspect session sandboxes.""" -This module handles: -- Creating sandboxes from filesystem snapshots -- Taking snapshots for session persistence - -Updated: 2026-01-15 to fix Sandbox.create API -""" - -import asyncio -import json -import secrets import time -from dataclasses import dataclass from typing import Any import modal @@ -38,44 +26,60 @@ from sandbox_runtime.log_config import get_logger from sandbox_runtime.types import SandboxStatus, SessionConfig -from ..app import app from ..app_config import APP_NAME -from ..images.base import base_image +from .launch import ( + ACCESS_PASSWORD_READ_TIMEOUT_SECONDS, + BaseImageSource, + RepositoryImageSource, + RepositoryImageUnavailableError, + SandboxImageSource, + SandboxLauncher, + SandboxLaunchSpec, + SnapshotImageSource, +) from .launch_policy import ( PENDING_VM_REFERENCE_PREFIX, ModalBackend, docker_allocation_name, docker_allocation_tags, - docker_base_image, - docker_runtime_env, - launch_kwargs, - parse_launch, parse_pending_vm_reference, ) -from .vcs_env import inject_vcs_env_vars +from .models import DEFAULT_VNC_ENABLED, SandboxConfig, SandboxHandle +from .tunnels import MAX_TUNNEL_PORTS + +# Preserve the existing public imports after moving their implementations. +__all__ = [ + "ACCESS_PASSWORD_READ_TIMEOUT_SECONDS", + "APP_NAME", + "CODE_SERVER_PORT", + "CODE_SERVER_PORT_ENV_VAR", + "CONTROL_TIMEOUT_SECONDS", + "DEFAULT_SANDBOX_TIMEOUT_SECONDS", + "DEFAULT_VNC_ENABLED", + "DOCKER_ENABLED_ENV_VAR", + "EXPECTED_TUNNEL_PORTS_ENV_VAR", + "MAX_TUNNEL_PORTS", + "NOVNC_PORT", + "NOVNC_PORT_ENV_VAR", + "PENDING_VM_REFERENCE_PREFIX", + "SANDBOX_TIMEOUT_ENV_VAR", + "SNAPSHOT_FILESYSTEM_TIMEOUT_SECONDS", + "TTYD_PROXY_PORT", + "TTYD_PROXY_PORT_ENV_VAR", + "TUNNEL_ENV_FILE_PATH", + "TUNNEL_ENV_SANDBOX_ID_KEY", + "VNC_PASSWORD_ENV_VAR", + "VNC_PASSWORD_MAX_BYTES", + "VNC_PORT", + "RepositoryImageUnavailableError", + "SandboxConfig", + "SandboxHandle", + "SandboxManager", +] log = get_logger("manager") SNAPSHOT_FILESYSTEM_TIMEOUT_SECONDS = 300 -ACCESS_PASSWORD_READ_TIMEOUT_SECONDS = 30 -MAX_TUNNEL_PORTS = 10 -DEFAULT_VNC_ENABLED = False -_RESERVED_LAUNCH_ENV_VARS = { - "RESTORED_FROM_SNAPSHOT", - "FROM_REPO_IMAGE", - "REPO_IMAGE_SHA", - "IMAGE_BUILD_MODE", - "TERMINAL_ENABLED", - "AGENT_SLACK_NOTIFY_ENABLED", - "SESSION_CONFIG", - VNC_PASSWORD_ENV_VAR, - NOVNC_PORT_ENV_VAR, - DOCKER_ENABLED_ENV_VAR, -} - - -class RepositoryImageUnavailableError(RuntimeError): - """The selected repository image no longer exists in Modal.""" class PendingVMReferenceNotVisible(RuntimeError): @@ -90,574 +94,13 @@ def _has_repository(repo_owner: str | None, repo_name: str | None) -> bool: return has_owner -async def _create_sandbox( - create_kwargs: dict[str, Any], *, repository_image: bool -) -> modal.Sandbox: - """The one `Sandbox.create` call; only its own NotFound means the image is gone.""" - try: - return await modal.Sandbox.create.aio( - "python", - "-m", - "sandbox_runtime.entrypoint", - **create_kwargs, - ) - except modal.exception.NotFoundError as e: - if repository_image: - raise RepositoryImageUnavailableError("repository image is unavailable") from e - raise - - -def _session_identity(session_config: SessionConfig | dict[str, Any] | None) -> str: - """The control-plane session id carried in the launch's session config.""" - if isinstance(session_config, dict): - session_id = session_config.get("session_id") - elif session_config is not None: - session_id = session_config.session_id - else: - session_id = None - return session_id if isinstance(session_id, str) else "" - - -@dataclass -class SandboxConfig: - """Configuration for creating a sandbox.""" - - repo_owner: str | None - repo_name: str | None - sandbox_backend: ModalBackend = "modal" - sandbox_id: str | None = None # Expected sandbox ID from control plane - session_config: SessionConfig | dict[str, Any] | None = None - control_plane_url: str = "" - sandbox_auth_token: str = "" - timeout_seconds: int = DEFAULT_SANDBOX_TIMEOUT_SECONDS - user_env_vars: dict[str, str] | None = None # User-provided env vars (repo secrets) - repo_image_id: str | None = None # Pre-built repo image ID from provider - repo_image_sha: str | None = None # Git SHA the repo image was built from - code_server_enabled: bool = False # Whether to start code-server in the sandbox - vnc_enabled: bool = DEFAULT_VNC_ENABLED # Whether to start the browser-accessible VNC desktop - agent_slack_notify_enabled: bool = ( - False # Whether to install the agent-initiated slack-notify tool - ) - settings: dict[str, Any] | None = ( - None # Sandbox settings (tunnelPorts, etc.) from control plane - ) - # A previous generation's sandbox id whose Docker VM may still be running - # after an ambiguous create (the control plane lost the response). Only - # Docker launches act on it; the named allocation is retired if owned. - retire_sandbox_id: str | None = None - launch_deadline_at_ms: int | None = None - - -@dataclass -class SandboxHandle: - """Handle to a sandbox.""" - - sandbox_id: str - modal_sandbox: modal.Sandbox - status: SandboxStatus - created_at: float - snapshot_id: str | None = None - modal_object_id: str | None = None # Modal's internal sandbox ID for API calls - code_server_url: str | None = None - code_server_password: str | None = None - vnc_url: str | None = None - vnc_password: str | None = None - ttyd_url: str | None = None # proxy tunnel URL (not ttyd directly) - tunnel_urls: dict[int, str] | None = None # port -> tunnel URL mapping for extra ports - sandbox_backend: ModalBackend = "modal" - - -@dataclass(frozen=True) -class _BaseImageSource: - pass - - -@dataclass(frozen=True) -class _RepositoryImageSource: - image_id: str - sha: str | None - - -@dataclass(frozen=True) -class _SnapshotImageSource: - image_id: str - clone_token: str | None - - -type _SandboxImageSource = _BaseImageSource | _RepositoryImageSource | _SnapshotImageSource - - -@dataclass(frozen=True) -class _SandboxLaunchSpec: - """Canonical launch configuration paired with one image source variant.""" - - config: SandboxConfig - source: _SandboxImageSource - - class SandboxManager: - """ - Manages sandbox lifecycle for Open-Inspect sessions. + """Normalize create/restore requests and manage existing provider sandboxes. - Responsibilities: - - Create sandboxes from snapshots or fresh images - - Take snapshots for session persistence + Launch translation and networking are owned by provider-local collaborators. + Session readiness and checkpoint/shutdown policy remain in the control plane. """ - @staticmethod - def _generate_code_server_password() -> str: - """Generate a random code-server password.""" - return secrets.token_urlsafe(16) - - @staticmethod - def _generate_vnc_password() -> str: - """Generate a random VNC password.""" - return secrets.token_urlsafe(VNC_PASSWORD_MAX_BYTES)[:VNC_PASSWORD_MAX_BYTES] - - @staticmethod - async def _resolve_tunnels( - sandbox: modal.Sandbox, - sandbox_id: str, - ports: list[int], - retries: int = 3, - backoff: float = 1.0, - ) -> dict[int, str]: - """Resolve tunnel URLs for the given ports from Modal, retrying on failure.""" - resolved: dict[int, str] = {} - for attempt in range(retries): - try: - loop = asyncio.get_running_loop() - tunnels = await loop.run_in_executor(None, sandbox.tunnels) - for port in ports: - if port in tunnels and port not in resolved: - resolved[port] = tunnels[port].url - log.info( - "tunnel.resolved", - sandbox_id=sandbox_id, - port=port, - url=tunnels[port].url, - ) - if len(resolved) == len(ports): - return resolved - except Exception as e: - log.warn( - "tunnel.resolve_error", - sandbox_id=sandbox_id, - attempt=attempt + 1, - retries=retries, - error=type(e).__name__, - exc=e, - ) - if attempt < retries - 1: - await asyncio.sleep(backoff * (attempt + 1)) - return resolved - - @staticmethod - def _validate_ports(raw: list) -> list[int]: - """Validate and sanitize tunnel ports: must be int, 1-65535, max MAX_TUNNEL_PORTS.""" - ports: list[int] = [] - for p in raw: - if isinstance(p, int) and 1 <= p <= 65535: - ports.append(p) - if len(ports) >= MAX_TUNNEL_PORTS: - break - return ports - - @staticmethod - def _resolve_service_ports(settings: dict[str, Any] | None) -> tuple[int, int, int]: - """Return effective (code_server_port, novnc_port, ttyd_proxy_port) from settings. - - Falls back to the service defaults when unset or invalid. The control - plane validates these before they reach here. - """ - s = settings or {} - - def coerce(value: Any, default: int) -> int: - if isinstance(value, int) and not isinstance(value, bool) and 1 <= value <= 65535: - return value - return default - - return ( - coerce(s.get("codeServerPort"), CODE_SERVER_PORT), - coerce(s.get("vncPort"), NOVNC_PORT), - coerce(s.get("terminalPort"), TTYD_PROXY_PORT), - ) - - @staticmethod - def _collect_exposed_ports( - code_server_enabled: bool, - vnc_enabled: bool, - terminal_enabled: bool, - settings: dict[str, Any] | None, - code_server_port: int, - novnc_port: int, - ttyd_proxy_port: int, - ) -> tuple[list[int], list[int]]: - """Return (all_exposed_ports, extra_tunnel_ports) from settings and feature flags.""" - # Raw VNC is localhost-only and must never be exposed, including as a - # user-configured extra tunnel. - reserved: set[int] = {VNC_PORT} - exposed: list[int] = [] - if code_server_enabled: - exposed.append(code_server_port) - reserved.add(code_server_port) - if vnc_enabled: - exposed.append(novnc_port) - reserved.add(novnc_port) - if terminal_enabled: - exposed.append(ttyd_proxy_port) - reserved.add(ttyd_proxy_port) - - raw_ports = (settings or {}).get("tunnelPorts", []) - tunnel_ports = SandboxManager._validate_ports(raw_ports) if raw_ports else [] - # Remove reserved ports from tunnel_ports to avoid duplicates - tunnel_ports = [p for p in tunnel_ports if p not in reserved] - exposed.extend(tunnel_ports) - return exposed, tunnel_ports - - @staticmethod - async def _resolve_and_setup_tunnels( - sandbox: modal.Sandbox, - sandbox_id: str, - code_server_enabled: bool, - vnc_enabled: bool, - terminal_enabled: bool, - extra_ports: list[int], - code_server_port: int, - novnc_port: int, - ttyd_proxy_port: int, - ) -> tuple[str | None, str | None, str | None, dict[int, str] | None]: - """Return (code_server_url, vnc_url, ttyd_url, extra_urls).""" - all_ports: list[int] = [] - if code_server_enabled: - all_ports.append(code_server_port) - if vnc_enabled: - all_ports.append(novnc_port) - if terminal_enabled: - all_ports.append(ttyd_proxy_port) - all_ports.extend(extra_ports) - - if not all_ports: - return None, None, None, None - - resolved = await SandboxManager._resolve_tunnels(sandbox, sandbox_id, all_ports) - - # Only pull a service port out of the resolved map when that service owns - # it. Otherwise a user's own port (e.g. 8080 with code-server disabled) - # would be misrouted to code_server_url and dropped from the tunnel map. - code_server_url = resolved.pop(code_server_port, None) if code_server_enabled else None - vnc_url = resolved.pop(novnc_port, None) if vnc_enabled else None - ttyd_url = resolved.pop(ttyd_proxy_port, None) if terminal_enabled else None - extra_urls = resolved if resolved else None - - if extra_urls: - await SandboxManager._write_tunnel_env_file(sandbox, sandbox_id, extra_urls) - - return code_server_url, vnc_url, ttyd_url, extra_urls - - @staticmethod - async def _write_tunnel_env_file( - sandbox: modal.Sandbox, - sandbox_id: str, - tunnel_urls: dict[int, str], - ) -> None: - """Write tunnel URLs to TUNNEL_ENV_FILE_PATH as a dotenv file. - - The first line tags the file with this sandbox's ID so the supervisor's - stale-file cleanup can tell a fresh write (this write can land before - the entrypoint runs) from a snapshot/image leftover. - - Failures are logged but do not block sandbox creation; URLs are also - returned to the control plane via the SandboxHandle. - """ - lines = [f"{TUNNEL_ENV_SANDBOX_ID_KEY}={sandbox_id}"] - lines += [f"TUNNEL_{port}={url}" for port, url in sorted(tunnel_urls.items())] - content = "\n".join(lines) + "\n" - try: - await sandbox.filesystem.write_text.aio(content, TUNNEL_ENV_FILE_PATH) - log.info( - "tunnel.urls_written", - sandbox_id=sandbox_id, - path=TUNNEL_ENV_FILE_PATH, - ports=list(tunnel_urls.keys()), - ) - except Exception as e: - log.warn( - "tunnel.urls_write_failed", - sandbox_id=sandbox_id, - path=TUNNEL_ENV_FILE_PATH, - exc=e, - ) - - async def _launch_sandbox(self, spec: _SandboxLaunchSpec) -> SandboxHandle: - """Launch a Modal sandbox from a normalized create or restore specification.""" - config = spec.config - has_repository = bool(config.repo_owner) - sandbox_id = config.sandbox_id - if not sandbox_id: - sandbox_name = ( - f"{config.repo_owner}-{config.repo_name}" if has_repository else "no-repository" - ) - sandbox_id = f"sandbox-{sandbox_name}-{int(time.time() * 1000)}" - - docker = parse_launch(config.sandbox_backend, config.settings) - env_vars = { - key: value - for key, value in (config.user_env_vars or {}).items() - if key not in _RESERVED_LAUNCH_ENV_VARS - } - env_vars.update( - { - "PYTHONUNBUFFERED": "1", - "SANDBOX_ID": sandbox_id, - "CONTROL_PLANE_URL": config.control_plane_url, - "SANDBOX_AUTH_TOKEN": config.sandbox_auth_token, - SANDBOX_TIMEOUT_ENV_VAR: str(config.timeout_seconds), - "REPO_OWNER": config.repo_owner or "", - "REPO_NAME": config.repo_name or "", - **docker_runtime_env(docker), - } - ) - - clone_token: str | None = None - include_github_cli_aliases = False - snapshot_id: str | None = None - if isinstance(spec.source, _BaseImageSource): - image = docker_base_image() if docker.enabled else base_image - elif isinstance(spec.source, _RepositoryImageSource): - image = modal.Image.from_id(spec.source.image_id) - env_vars["FROM_REPO_IMAGE"] = "true" - env_vars["REPO_IMAGE_SHA"] = spec.source.sha or "" - else: - image = modal.Image.from_id(spec.source.image_id) - env_vars["RESTORED_FROM_SNAPSHOT"] = "true" - clone_token = spec.source.clone_token - include_github_cli_aliases = True - snapshot_id = spec.source.image_id - - if config.session_config is not None: - env_vars["SESSION_CONFIG"] = ( - json.dumps(config.session_config) - if isinstance(config.session_config, dict) - else config.session_config.model_dump_json() - ) - - inject_vcs_env_vars( - env_vars, - clone_token=clone_token if has_repository else None, - include_github_cli_aliases=include_github_cli_aliases, - ) - - code_server_password: str | None = None - if config.code_server_enabled: - code_server_password = self._generate_code_server_password() - env_vars["CODE_SERVER_PASSWORD"] = code_server_password - - vnc_password: str | None = None - if config.vnc_enabled: - vnc_password = self._generate_vnc_password() - env_vars[VNC_PASSWORD_ENV_VAR] = vnc_password - - terminal_enabled = bool((config.settings or {}).get("terminalEnabled", False)) - if terminal_enabled: - env_vars["TERMINAL_ENABLED"] = "true" - if config.agent_slack_notify_enabled: - env_vars["AGENT_SLACK_NOTIFY_ENABLED"] = "true" - - code_server_port, novnc_port, ttyd_proxy_port = self._resolve_service_ports(config.settings) - if config.code_server_enabled: - env_vars[CODE_SERVER_PORT_ENV_VAR] = str(code_server_port) - if config.vnc_enabled: - env_vars[NOVNC_PORT_ENV_VAR] = str(novnc_port) - if terminal_enabled: - env_vars[TTYD_PROXY_PORT_ENV_VAR] = str(ttyd_proxy_port) - - exposed_ports, tunnel_ports = self._collect_exposed_ports( - config.code_server_enabled, - config.vnc_enabled, - terminal_enabled, - config.settings, - code_server_port, - novnc_port, - ttyd_proxy_port, - ) - if tunnel_ports: - env_vars[EXPECTED_TUNNEL_PORTS_ENV_VAR] = ",".join(str(p) for p in tunnel_ports) - - # from_name handles cache their resolved ID; use a fresh handle on every - # launch so a deleted and recreated secret can be resolved again. - llm_secrets = modal.Secret.from_name("llm-api-keys") - await llm_secrets.hydrate.aio() - - create_kwargs: dict[str, Any] = { - "image": image, - "app": app, - "secrets": [llm_secrets], - "timeout": config.timeout_seconds, - "workdir": "/workspace", - "env": env_vars, - **launch_kwargs(docker), - } - if exposed_ports: - create_kwargs["encrypted_ports"] = exposed_ports - - repository_image = isinstance(spec.source, _RepositoryImageSource) - if docker.enabled: - sandbox, adopted = await self._launch_docker_sandbox( - session_id=_session_identity(config.session_config), - sandbox_id=sandbox_id, - retire_sandbox_id=config.retire_sandbox_id, - create_kwargs=create_kwargs, - repository_image=repository_image, - launch_deadline_at_ms=config.launch_deadline_at_ms, - ) - if adopted: - passwords = await self._read_access_passwords( - sandbox, - code_server_enabled=config.code_server_enabled, - vnc_enabled=config.vnc_enabled, - ) - code_server_password = passwords.get("CODE_SERVER_PASSWORD") - vnc_password = passwords.get(VNC_PASSWORD_ENV_VAR) - else: - sandbox = await _create_sandbox(create_kwargs, repository_image=repository_image) - modal_object_id = sandbox.object_id - ( - code_server_url, - vnc_url, - ttyd_url, - extra_tunnel_urls, - ) = await self._resolve_and_setup_tunnels( - sandbox, - sandbox_id, - config.code_server_enabled, - config.vnc_enabled, - terminal_enabled, - tunnel_ports, - code_server_port, - novnc_port, - ttyd_proxy_port, - ) - - return SandboxHandle( - sandbox_id=sandbox_id, - modal_sandbox=sandbox, - status=SandboxStatus.WARMING, - created_at=time.time(), - snapshot_id=snapshot_id, - modal_object_id=modal_object_id, - code_server_url=code_server_url, - code_server_password=code_server_password, - vnc_url=vnc_url, - vnc_password=vnc_password, - ttyd_url=ttyd_url, - tunnel_urls=extra_tunnel_urls, - sandbox_backend=docker.backend, - ) - - async def _launch_docker_sandbox( - self, - *, - session_id: str, - sandbox_id: str, - retire_sandbox_id: str | None, - create_kwargs: dict[str, Any], - repository_image: bool, - launch_deadline_at_ms: int | None = None, - ) -> tuple[modal.Sandbox, bool]: - """Create a Docker VM under a deterministic name, adopting an existing one. - - VM creation can outlive the control plane's HTTP request. One name per - session serializes generations at Modal even when a predecessor lookup - misses an in-flight create. Only matching generation tags permit adoption. - """ - if retire_sandbox_id: - await self._retire_docker_allocation(session_id, retire_sandbox_id) - name = docker_allocation_name(session_id) - tags = docker_allocation_tags(session_id, sandbox_id) - existing = await self._find_owned_docker_allocation(name, tags) - if existing is None: - if launch_deadline_at_ms is not None and time.time() * 1000 >= launch_deadline_at_ms: - raise RuntimeError("VM launch deadline expired") - try: - sandbox = await _create_sandbox( - {**create_kwargs, "name": name, "tags": tags}, - repository_image=repository_image, - ) - return sandbox, False - except modal.exception.AlreadyExistsError: - existing = await self._find_owned_docker_allocation(name, tags) - if existing is None: - raise - log.info( - "sandbox.docker_allocation_adopted", - sandbox_id=sandbox_id, - modal_object_id=existing.object_id, - ) - return existing, True - - @staticmethod - async def _read_access_passwords( - sandbox: modal.Sandbox, *, code_server_enabled: bool, vnc_enabled: bool - ) -> dict[str, str]: - """Recover only enabled service credentials from the owned VM's launch environment.""" - keys = [] - if code_server_enabled: - keys.append("CODE_SERVER_PASSWORD") - if vnc_enabled: - keys.append(VNC_PASSWORD_ENV_VAR) - if not keys: - return {} - process = await sandbox.exec.aio( - "python", - "-I", - "-c", - "import json, os, sys; print(json.dumps({k: os.environ.get(k) for k in sys.argv[1:]}))", - *keys, - timeout=ACCESS_PASSWORD_READ_TIMEOUT_SECONDS, - ) - output = await process.stdout.read.aio() - if await process.wait.aio() != 0: - raise RuntimeError("Could not recover adopted sandbox access credentials") - try: - passwords = json.loads(output) - except ValueError: - raise RuntimeError("Could not recover adopted sandbox access credentials") from None - if not isinstance(passwords, dict) or any( - not isinstance(passwords.get(key), str) or not passwords[key] for key in keys - ): - raise RuntimeError("Could not recover adopted sandbox access credentials") - return {key: passwords[key] for key in keys} - - @staticmethod - async def _find_owned_docker_allocation( - name: str, tags: dict[str, str] - ) -> modal.Sandbox | None: - try: - sandbox = await modal.Sandbox.from_name.aio(APP_NAME, name) - except modal.exception.NotFoundError: - return None - if await sandbox.get_tags.aio() != tags: - raise RuntimeError("Docker sandbox allocation ownership mismatch") - return sandbox - - async def _retire_docker_allocation(self, session_id: str, sandbox_id: str) -> None: - """Terminate a prior generation's named VM, only when its ownership tags match.""" - name = docker_allocation_name(session_id) - try: - sandbox = await modal.Sandbox.from_name.aio(APP_NAME, name) - except modal.exception.NotFoundError: - return - if await sandbox.get_tags.aio() != docker_allocation_tags(session_id, sandbox_id): - log.warn("sandbox.docker_allocation_retire_mismatch", sandbox_id=sandbox_id) - return - await sandbox.terminate.aio(wait=True) - log.info( - "sandbox.docker_allocation_retired", - sandbox_id=sandbox_id, - modal_object_id=sandbox.object_id, - ) - async def create_sandbox( self, config: SandboxConfig, @@ -667,7 +110,7 @@ async def create_sandbox( Creates from the pre-built repo image when one is provided, otherwise from the base image. Snapshot restores go through - restore_sandbox, not this path. + restore_from_snapshot, not this path. Args: config: Sandbox configuration including repo info and session config @@ -679,14 +122,14 @@ async def create_sandbox( _has_repository(config.repo_owner, config.repo_name) if config.repo_image_id: - source: _SandboxImageSource = _RepositoryImageSource( + source: SandboxImageSource = RepositoryImageSource( image_id=config.repo_image_id, sha=config.repo_image_sha, ) else: - source = _BaseImageSource() + source = BaseImageSource() - handle = await self._launch_sandbox(_SandboxLaunchSpec(config=config, source=source)) + handle = await SandboxLauncher().launch(SandboxLaunchSpec(config=config, source=source)) duration_ms = int((time.time() - start_time) * 1000) log.info( @@ -800,7 +243,9 @@ async def get_sandbox_by_id(self, sandbox_id: str) -> SandboxHandle | None: APP_NAME, docker_allocation_name(identity[0]) ) except modal.exception.NotFoundError: - raise PendingVMReferenceNotVisible("VM launch identity is not yet visible") + raise PendingVMReferenceNotVisible( + "VM launch identity is not yet visible" + ) from None else: try: modal_sandbox = await modal.Sandbox.from_id.aio(sandbox_id) @@ -817,7 +262,7 @@ async def get_sandbox_by_id(self, sandbox_id: str) -> SandboxHandle | None: sandbox_id=sandbox_id, modal_object_id=modal_sandbox.object_id, modal_sandbox=modal_sandbox, - status=SandboxStatus.READY, # Assume ready if we can retrieve it + status=SandboxStatus.READY, created_at=time.time(), ) @@ -874,8 +319,8 @@ async def restore_from_snapshot( # so the gh CLI keeps working on snapshots predating the gh wrapper. # Host scoping remains common with fresh creates. These compatibility # credentials are explicitly requested only by the restore path. - handle = await self._launch_sandbox( - _SandboxLaunchSpec( + handle = await SandboxLauncher().launch( + SandboxLaunchSpec( config=SandboxConfig( repo_owner=repo_owner, repo_name=repo_name, @@ -888,12 +333,12 @@ async def restore_from_snapshot( code_server_enabled=code_server_enabled, vnc_enabled=vnc_enabled, agent_slack_notify_enabled=agent_slack_notify_enabled, - retire_sandbox_id=retire_sandbox_id, settings=settings, + retire_sandbox_id=retire_sandbox_id, sandbox_backend=sandbox_backend, launch_deadline_at_ms=launch_deadline_at_ms, ), - source=_SnapshotImageSource( + source=SnapshotImageSource( image_id=snapshot_image_id, clone_token=clone_token, ), diff --git a/packages/modal-infra/src/sandbox/models.py b/packages/modal-infra/src/sandbox/models.py new file mode 100644 index 0000000000..61125ecaa1 --- /dev/null +++ b/packages/modal-infra/src/sandbox/models.py @@ -0,0 +1,59 @@ +"""Configuration and handles shared by Modal sandbox operations.""" + +from dataclasses import dataclass +from typing import Any + +import modal + +from sandbox_runtime.constants import DEFAULT_SANDBOX_TIMEOUT_SECONDS +from sandbox_runtime.types import SandboxStatus, SessionConfig + +from .launch_policy import ModalBackend + +DEFAULT_VNC_ENABLED = False + + +@dataclass +class SandboxConfig: + """Configuration for creating a sandbox.""" + + repo_owner: str | None + repo_name: str | None + sandbox_id: str | None = None # Expected sandbox ID from control plane + session_config: SessionConfig | dict[str, Any] | None = None + control_plane_url: str = "" + sandbox_auth_token: str = "" + timeout_seconds: int = DEFAULT_SANDBOX_TIMEOUT_SECONDS + user_env_vars: dict[str, str] | None = None # User-provided env vars (repo secrets) + repo_image_id: str | None = None # Pre-built repo image ID from provider + repo_image_sha: str | None = None # Git SHA the repo image was built from + code_server_enabled: bool = False # Whether to start code-server in the sandbox + vnc_enabled: bool = DEFAULT_VNC_ENABLED # Whether to start the browser-accessible VNC desktop + agent_slack_notify_enabled: bool = ( + False # Whether to install the agent-initiated slack-notify tool + ) + settings: dict[str, Any] | None = ( + None # Sandbox settings (tunnelPorts, etc.) from control plane + ) + sandbox_backend: ModalBackend = "modal" + retire_sandbox_id: str | None = None + launch_deadline_at_ms: int | None = None + + +@dataclass +class SandboxHandle: + """Handle to a sandbox.""" + + sandbox_id: str + modal_sandbox: modal.Sandbox + status: SandboxStatus + created_at: float + snapshot_id: str | None = None + modal_object_id: str | None = None # Modal's internal sandbox ID for API calls + code_server_url: str | None = None + code_server_password: str | None = None + vnc_url: str | None = None + vnc_password: str | None = None + ttyd_url: str | None = None # proxy tunnel URL (not ttyd directly) + tunnel_urls: dict[int, str] | None = None # port -> tunnel URL mapping for extra ports + sandbox_backend: ModalBackend = "modal" diff --git a/packages/modal-infra/src/sandbox/tunnels.py b/packages/modal-infra/src/sandbox/tunnels.py new file mode 100644 index 0000000000..17210838aa --- /dev/null +++ b/packages/modal-infra/src/sandbox/tunnels.py @@ -0,0 +1,215 @@ +"""Service port ownership and best-effort Modal tunnel publication.""" + +import asyncio +from typing import Any, NamedTuple + +import modal + +from sandbox_runtime.constants import ( + CODE_SERVER_PORT, + CODE_SERVER_PORT_ENV_VAR, + EXPECTED_TUNNEL_PORTS_ENV_VAR, + NOVNC_PORT, + NOVNC_PORT_ENV_VAR, + TTYD_PROXY_PORT, + TTYD_PROXY_PORT_ENV_VAR, + TUNNEL_ENV_FILE_PATH, + TUNNEL_ENV_SANDBOX_ID_KEY, + VNC_PORT, +) +from sandbox_runtime.log_config import get_logger + +# Preserve the logger name used by existing launch/tunnel dashboards. +log = get_logger("manager") +MAX_TUNNEL_PORTS = 10 +DEFAULT_TUNNEL_RESOLUTION_RETRIES = 3 +DEFAULT_TUNNEL_RESOLUTION_BACKOFF_SECONDS = 1.0 + + +class TunnelUrls(NamedTuple): + """Resolved service URLs and any user-requested tunnels.""" + + code_server_url: str | None = None + vnc_url: str | None = None + ttyd_url: str | None = None + tunnel_urls: dict[int, str] | None = None + + +class SandboxTunnels: + """Keep exposed ports, runtime environment, and URL routing in agreement. + + Service ownership is resolved once for a launch. Disabled service ports + remain available as user tunnels; raw VNC is never an extra tunnel. + """ + + def __init__( + self, + *, + code_server_enabled: bool = False, + vnc_enabled: bool = False, + settings: dict[str, Any] | None = None, + ) -> None: + settings = settings or {} + code_server_port, novnc_port, ttyd_proxy_port = self._resolve_service_ports(settings) + self._code_server_port = code_server_port if code_server_enabled else None + self._novnc_port = novnc_port if vnc_enabled else None + self._ttyd_proxy_port = ( + ttyd_proxy_port if bool(settings.get("terminalEnabled", False)) else None + ) + service_ports = [ + port + for port in (self._code_server_port, self._novnc_port, self._ttyd_proxy_port) + if port is not None + ] + reserved = {VNC_PORT, *service_ports} + raw_ports = settings.get("tunnelPorts", []) + self._extra_ports = ( + [port for port in self._validate_ports(raw_ports) if port not in reserved] + if raw_ports + else [] + ) + self.exposed_ports = service_ports + self._extra_ports + + @property + def environment(self) -> dict[str, str]: + """Runtime settings derived from the same ports used for exposure.""" + env: dict[str, str] = {} + if self._code_server_port is not None: + env[CODE_SERVER_PORT_ENV_VAR] = str(self._code_server_port) + if self._novnc_port is not None: + env[NOVNC_PORT_ENV_VAR] = str(self._novnc_port) + if self._ttyd_proxy_port is not None: + env["TERMINAL_ENABLED"] = "true" + env[TTYD_PROXY_PORT_ENV_VAR] = str(self._ttyd_proxy_port) + if self._extra_ports: + env[EXPECTED_TUNNEL_PORTS_ENV_VAR] = ",".join(str(p) for p in self._extra_ports) + return env + + async def resolve(self, sandbox: modal.Sandbox, sandbox_id: str) -> TunnelUrls: + """Resolve URLs and publish extras; partial resolution/write failures are non-fatal.""" + if not self.exposed_ports: + return TunnelUrls() + + resolved = await self._resolve_tunnels(sandbox, sandbox_id, self.exposed_ports) + # A disabled service does not own its default port: leave it in extras. + code_server_url = ( + resolved.pop(self._code_server_port, None) + if self._code_server_port is not None + else None + ) + vnc_url = resolved.pop(self._novnc_port, None) if self._novnc_port is not None else None + ttyd_url = ( + resolved.pop(self._ttyd_proxy_port, None) if self._ttyd_proxy_port is not None else None + ) + extra_urls = resolved or None + if extra_urls: + await self._write_tunnel_env_file(sandbox, sandbox_id, extra_urls) + return TunnelUrls( + code_server_url=code_server_url, + vnc_url=vnc_url, + ttyd_url=ttyd_url, + tunnel_urls=extra_urls, + ) + + @staticmethod + async def _resolve_tunnels( + sandbox: modal.Sandbox, + sandbox_id: str, + ports: list[int], + retries: int = DEFAULT_TUNNEL_RESOLUTION_RETRIES, + backoff_seconds: float = DEFAULT_TUNNEL_RESOLUTION_BACKOFF_SECONDS, + ) -> dict[int, str]: + """Resolve tunnel URLs for the given ports from Modal, retrying on failure.""" + resolved: dict[int, str] = {} + for attempt in range(retries): + try: + loop = asyncio.get_running_loop() + tunnels = await loop.run_in_executor(None, sandbox.tunnels) + for port in ports: + if port in tunnels and port not in resolved: + resolved[port] = tunnels[port].url + log.info( + "tunnel.resolved", + sandbox_id=sandbox_id, + port=port, + url=tunnels[port].url, + ) + if len(resolved) == len(ports): + return resolved + except Exception as e: + log.warn( + "tunnel.resolve_error", + sandbox_id=sandbox_id, + attempt=attempt + 1, + retries=retries, + error=type(e).__name__, + exc=e, + ) + if attempt < retries - 1: + await asyncio.sleep(backoff_seconds * (attempt + 1)) + return resolved + + @staticmethod + def _validate_ports(raw: list[Any]) -> list[int]: + """Validate and sanitize tunnel ports: must be int, 1-65535, max MAX_TUNNEL_PORTS.""" + ports: list[int] = [] + for p in raw: + if isinstance(p, int) and not isinstance(p, bool) and 1 <= p <= 65535: + ports.append(p) + if len(ports) >= MAX_TUNNEL_PORTS: + break + return ports + + @staticmethod + def _resolve_service_ports(settings: dict[str, Any] | None) -> tuple[int, int, int]: + """Return effective (code_server_port, novnc_port, ttyd_proxy_port) from settings. + + Falls back to the service defaults when unset or invalid. The control + plane validates these before they reach here. + """ + s = settings or {} + + def coerce(value: Any, default: int) -> int: + if isinstance(value, int) and not isinstance(value, bool) and 1 <= value <= 65535: + return value + return default + + return ( + coerce(s.get("codeServerPort"), CODE_SERVER_PORT), + coerce(s.get("vncPort"), NOVNC_PORT), + coerce(s.get("terminalPort"), TTYD_PROXY_PORT), + ) + + @staticmethod + async def _write_tunnel_env_file( + sandbox: modal.Sandbox, + sandbox_id: str, + tunnel_urls: dict[int, str], + ) -> None: + """Write tunnel URLs to TUNNEL_ENV_FILE_PATH as a dotenv file. + + The first line tags the file with this sandbox's ID so the supervisor's + stale-file cleanup can tell a fresh write (this write can land before + the entrypoint runs) from a snapshot/image leftover. + + Failures are logged but do not block sandbox creation; URLs are also + returned to the control plane via the SandboxHandle. + """ + lines = [f"{TUNNEL_ENV_SANDBOX_ID_KEY}={sandbox_id}"] + lines += [f"TUNNEL_{port}={url}" for port, url in sorted(tunnel_urls.items())] + content = "\n".join(lines) + "\n" + try: + await sandbox.filesystem.write_text.aio(content, TUNNEL_ENV_FILE_PATH) + log.info( + "tunnel.urls_written", + sandbox_id=sandbox_id, + path=TUNNEL_ENV_FILE_PATH, + ports=list(tunnel_urls.keys()), + ) + except Exception as e: + log.warn( + "tunnel.urls_write_failed", + sandbox_id=sandbox_id, + path=TUNNEL_ENV_FILE_PATH, + exc=e, + ) diff --git a/packages/modal-infra/tests/test_agent_slack_notify_env.py b/packages/modal-infra/tests/test_agent_slack_notify_env.py index abc20cce92..ede0a3c971 100644 --- a/packages/modal-infra/tests/test_agent_slack_notify_env.py +++ b/packages/modal-infra/tests/test_agent_slack_notify_env.py @@ -5,6 +5,7 @@ import pytest from src.sandbox.manager import SandboxConfig, SandboxManager +from src.sandbox.tunnels import SandboxTunnels, TunnelUrls def _patch_create(monkeypatch, captured: dict) -> None: @@ -21,11 +22,11 @@ class FakeSandbox: fake_create = MagicMock() fake_create.aio = fake_create_aio - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", fake_create) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", fake_create) monkeypatch.setattr( - SandboxManager, - "_resolve_and_setup_tunnels", - AsyncMock(return_value=(None, None, None, None)), + SandboxTunnels, + "resolve", + AsyncMock(return_value=TunnelUrls(None, None, None, None)), ) @@ -79,7 +80,7 @@ async def test_env_set_when_enabled(self, monkeypatch): class FakeImage: object_id = "img-123" - monkeypatch.setattr("src.sandbox.manager.modal.Image.from_id", lambda *a, **k: FakeImage()) + monkeypatch.setattr("src.sandbox.launch.modal.Image.from_id", lambda *a, **k: FakeImage()) _patch_create(monkeypatch, captured) manager = SandboxManager() diff --git a/packages/modal-infra/tests/test_code_server.py b/packages/modal-infra/tests/test_code_server.py index f950e2a31b..302d8c7675 100644 --- a/packages/modal-infra/tests/test_code_server.py +++ b/packages/modal-infra/tests/test_code_server.py @@ -4,25 +4,27 @@ import pytest +from src.sandbox.launch import SandboxLauncher from src.sandbox.manager import CODE_SERVER_PORT, SandboxConfig, SandboxManager +from src.sandbox.tunnels import SandboxTunnels, TunnelUrls class TestGenerateCodeServerPassword: - """SandboxManager._generate_code_server_password tests.""" + """SandboxLauncher._generate_code_server_password tests.""" def test_returns_nonempty_password(self): - password = SandboxManager._generate_code_server_password() + password = SandboxLauncher._generate_code_server_password() assert len(password) > 0 def test_generates_unique_passwords(self): passwords = set() for _ in range(20): - passwords.add(SandboxManager._generate_code_server_password()) + passwords.add(SandboxLauncher._generate_code_server_password()) assert len(passwords) == 20 class TestResolveCodeServerTunnel: - """SandboxManager._resolve_tunnels tests for code-server port.""" + """SandboxTunnels._resolve_tunnels tests for code-server port.""" @pytest.mark.asyncio async def test_returns_tunnel_url_on_success(self): @@ -32,7 +34,7 @@ async def test_returns_tunnel_url_on_success(self): sandbox = MagicMock() sandbox.tunnels.return_value = {CODE_SERVER_PORT: tunnel} - resolved = await SandboxManager._resolve_tunnels(sandbox, "sb-123", [CODE_SERVER_PORT]) + resolved = await SandboxTunnels._resolve_tunnels(sandbox, "sb-123", [CODE_SERVER_PORT]) assert resolved.get(CODE_SERVER_PORT) == "https://tunnel.example.com" @pytest.mark.asyncio @@ -40,9 +42,9 @@ async def test_returns_empty_on_exception_after_retries(self): sandbox = MagicMock() sandbox.tunnels.side_effect = Exception("tunnel unavailable") - with patch("src.sandbox.manager.asyncio.sleep", new_callable=AsyncMock): - resolved = await SandboxManager._resolve_tunnels( - sandbox, "sb-123", [CODE_SERVER_PORT], retries=2, backoff=0.0 + with patch("src.sandbox.tunnels.asyncio.sleep", new_callable=AsyncMock): + resolved = await SandboxTunnels._resolve_tunnels( + sandbox, "sb-123", [CODE_SERVER_PORT], retries=2, backoff_seconds=0.0 ) assert resolved == {} assert sandbox.tunnels.call_count == 2 @@ -52,9 +54,9 @@ async def test_returns_empty_when_port_missing_after_retries(self): sandbox = MagicMock() sandbox.tunnels.return_value = {} # no entry for CODE_SERVER_PORT - with patch("src.sandbox.manager.asyncio.sleep", new_callable=AsyncMock): - resolved = await SandboxManager._resolve_tunnels( - sandbox, "sb-123", [CODE_SERVER_PORT], retries=2, backoff=0.0 + with patch("src.sandbox.tunnels.asyncio.sleep", new_callable=AsyncMock): + resolved = await SandboxTunnels._resolve_tunnels( + sandbox, "sb-123", [CODE_SERVER_PORT], retries=2, backoff_seconds=0.0 ) assert resolved == {} @@ -69,9 +71,9 @@ async def test_retries_then_succeeds(self): {CODE_SERVER_PORT: tunnel}, ] - with patch("src.sandbox.manager.asyncio.sleep", new_callable=AsyncMock): - resolved = await SandboxManager._resolve_tunnels( - sandbox, "sb-123", [CODE_SERVER_PORT], retries=3, backoff=0.0 + with patch("src.sandbox.tunnels.asyncio.sleep", new_callable=AsyncMock): + resolved = await SandboxTunnels._resolve_tunnels( + sandbox, "sb-123", [CODE_SERVER_PORT], retries=3, backoff_seconds=0.0 ) assert resolved.get(CODE_SERVER_PORT) == "https://tunnel.example.com" assert sandbox.tunnels.call_count == 2 @@ -96,12 +98,12 @@ class FakeSandbox: fake_create = MagicMock() fake_create.aio = fake_create_aio - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", fake_create) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", fake_create) monkeypatch.setattr( - SandboxManager, - "_resolve_and_setup_tunnels", - AsyncMock(return_value=("https://cs.example.com", None, None, None)), + SandboxTunnels, + "resolve", + AsyncMock(return_value=TunnelUrls("https://cs.example.com", None, None, None)), ) manager = SandboxManager() @@ -140,10 +142,10 @@ class FakeSandbox: fake_create = MagicMock() fake_create.aio = fake_create_aio - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", fake_create) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", fake_create) - tunnel_mock = AsyncMock(return_value=(None, None, None, None)) - monkeypatch.setattr(SandboxManager, "_resolve_and_setup_tunnels", tunnel_mock) + tunnel_mock = AsyncMock(return_value=TunnelUrls(None, None, None, None)) + monkeypatch.setattr(SandboxTunnels, "resolve", tunnel_mock) manager = SandboxManager() config = SandboxConfig( @@ -187,12 +189,12 @@ class FakeSandbox: fake_create = MagicMock() fake_create.aio = fake_create_aio - monkeypatch.setattr("src.sandbox.manager.modal.Image.from_id", fake_from_id) - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", fake_create) + monkeypatch.setattr("src.sandbox.launch.modal.Image.from_id", fake_from_id) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", fake_create) monkeypatch.setattr( - SandboxManager, - "_resolve_and_setup_tunnels", - AsyncMock(return_value=("https://cs-restored.example.com", None, None, None)), + SandboxTunnels, + "resolve", + AsyncMock(return_value=TunnelUrls("https://cs-restored.example.com", None, None, None)), ) manager = SandboxManager() @@ -238,10 +240,10 @@ class FakeSandbox: fake_create = MagicMock() fake_create.aio = fake_create_aio - monkeypatch.setattr("src.sandbox.manager.modal.Image.from_id", fake_from_id) - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", fake_create) - tunnel_mock = AsyncMock(return_value=(None, None, None, None)) - monkeypatch.setattr(SandboxManager, "_resolve_and_setup_tunnels", tunnel_mock) + monkeypatch.setattr("src.sandbox.launch.modal.Image.from_id", fake_from_id) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", fake_create) + tunnel_mock = AsyncMock(return_value=TunnelUrls(None, None, None, None)) + monkeypatch.setattr(SandboxTunnels, "resolve", tunnel_mock) manager = SandboxManager() handle = await manager.restore_from_snapshot( diff --git a/packages/modal-infra/tests/test_llm_secrets.py b/packages/modal-infra/tests/test_llm_secrets.py index daf8935755..a6448ecc7b 100644 --- a/packages/modal-infra/tests/test_llm_secrets.py +++ b/packages/modal-infra/tests/test_llm_secrets.py @@ -26,7 +26,7 @@ class FakeSandbox: return FakeSandbox() fake_create_aio.aio = fake_create_aio - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", fake_create_aio) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", fake_create_aio) return captured @@ -44,7 +44,7 @@ async def test_restore_attaches_the_deployment_wide_secret( class FakeImage: object_id = "img-llm-secrets" - monkeypatch.setattr("src.sandbox.manager.modal.Image.from_id", lambda *a, **k: FakeImage()) + monkeypatch.setattr("src.sandbox.launch.modal.Image.from_id", lambda *a, **k: FakeImage()) await SandboxManager().restore_from_snapshot( snapshot_image_id="img-abc", diff --git a/packages/modal-infra/tests/test_manager_exports.py b/packages/modal-infra/tests/test_manager_exports.py new file mode 100644 index 0000000000..cddf8c6564 --- /dev/null +++ b/packages/modal-infra/tests/test_manager_exports.py @@ -0,0 +1,45 @@ +"""Compatibility coverage for the manager's pre-refactor constant imports.""" + +import pytest + +from sandbox_runtime import constants +from sandbox_runtime.docker_control import CONTROL_TIMEOUT_SECONDS +from src.app_config import APP_NAME +from src.sandbox import manager +from src.sandbox.launch import ACCESS_PASSWORD_READ_TIMEOUT_SECONDS +from src.sandbox.launch_policy import PENDING_VM_REFERENCE_PREFIX +from src.sandbox.models import DEFAULT_VNC_ENABLED +from src.sandbox.tunnels import MAX_TUNNEL_PORTS + + +@pytest.mark.parametrize( + "name, expected", + [ + ("ACCESS_PASSWORD_READ_TIMEOUT_SECONDS", ACCESS_PASSWORD_READ_TIMEOUT_SECONDS), + ("APP_NAME", APP_NAME), + ("CODE_SERVER_PORT", constants.CODE_SERVER_PORT), + ("CODE_SERVER_PORT_ENV_VAR", constants.CODE_SERVER_PORT_ENV_VAR), + ("DEFAULT_SANDBOX_TIMEOUT_SECONDS", constants.DEFAULT_SANDBOX_TIMEOUT_SECONDS), + ("DEFAULT_VNC_ENABLED", DEFAULT_VNC_ENABLED), + ("DOCKER_ENABLED_ENV_VAR", constants.DOCKER_ENABLED_ENV_VAR), + ("EXPECTED_TUNNEL_PORTS_ENV_VAR", constants.EXPECTED_TUNNEL_PORTS_ENV_VAR), + ("MAX_TUNNEL_PORTS", MAX_TUNNEL_PORTS), + ("NOVNC_PORT", constants.NOVNC_PORT), + ("NOVNC_PORT_ENV_VAR", constants.NOVNC_PORT_ENV_VAR), + ("PENDING_VM_REFERENCE_PREFIX", PENDING_VM_REFERENCE_PREFIX), + ("SANDBOX_TIMEOUT_ENV_VAR", constants.SANDBOX_TIMEOUT_ENV_VAR), + ("SNAPSHOT_FILESYSTEM_TIMEOUT_SECONDS", 300), + ("CONTROL_TIMEOUT_SECONDS", CONTROL_TIMEOUT_SECONDS), + ("TTYD_PROXY_PORT", constants.TTYD_PROXY_PORT), + ("TTYD_PROXY_PORT_ENV_VAR", constants.TTYD_PROXY_PORT_ENV_VAR), + ("TUNNEL_ENV_FILE_PATH", constants.TUNNEL_ENV_FILE_PATH), + ("TUNNEL_ENV_SANDBOX_ID_KEY", constants.TUNNEL_ENV_SANDBOX_ID_KEY), + ("VNC_PASSWORD_ENV_VAR", constants.VNC_PASSWORD_ENV_VAR), + ("VNC_PASSWORD_MAX_BYTES", constants.VNC_PASSWORD_MAX_BYTES), + ("VNC_PORT", constants.VNC_PORT), + ], +) +def test_legacy_manager_constant_exports(name, expected): + """Legacy import names retain their values and are explicitly public.""" + assert getattr(manager, name) == expected + assert name in manager.__all__ diff --git a/packages/modal-infra/tests/test_sandbox_env_vars.py b/packages/modal-infra/tests/test_sandbox_env_vars.py index 131ff0ef1f..be4130a938 100644 --- a/packages/modal-infra/tests/test_sandbox_env_vars.py +++ b/packages/modal-infra/tests/test_sandbox_env_vars.py @@ -8,6 +8,7 @@ VNC_PASSWORD_MAX_BYTES, ) from sandbox_runtime.types import SessionConfig +from src.sandbox.launch import SandboxLauncher from src.sandbox.manager import ( DEFAULT_SANDBOX_TIMEOUT_SECONDS, SandboxConfig, @@ -86,7 +87,7 @@ class FakeSandbox: return FakeSandbox() fake_create_aio.aio = fake_create_aio - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", fake_create_aio) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", fake_create_aio) manager = SandboxManager() config = SandboxConfig( @@ -132,8 +133,8 @@ class FakeSandbox: return FakeSandbox() fake_create_aio.aio = fake_create_aio - monkeypatch.setattr("src.sandbox.manager.modal.Image.from_id", fake_from_id) - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", fake_create_aio) + monkeypatch.setattr("src.sandbox.launch.modal.Image.from_id", fake_from_id) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", fake_create_aio) manager = SandboxManager() await manager.restore_from_snapshot( @@ -179,7 +180,7 @@ async def test_create_preserves_managed_provider_env_isolation( monkeypatch, managed_marker, suppressed_api_key ): captured = {} - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", _fake_sandbox_create(captured)) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", _fake_sandbox_create(captured)) await SandboxManager().create_sandbox( SandboxConfig( @@ -217,7 +218,7 @@ async def test_restore_preserves_managed_provider_env_isolation( def test_generated_vnc_password_respects_protocol_limit(): - assert len(SandboxManager._generate_vnc_password().encode()) == VNC_PASSWORD_MAX_BYTES + assert len(SandboxLauncher._generate_vnc_password().encode()) == VNC_PASSWORD_MAX_BYTES @pytest.mark.asyncio @@ -241,8 +242,8 @@ class FakeSandbox: return FakeSandbox() fake_create_aio.aio = fake_create_aio - monkeypatch.setattr("src.sandbox.manager.modal.Image.from_id", fake_from_id) - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", fake_create_aio) + monkeypatch.setattr("src.sandbox.launch.modal.Image.from_id", fake_from_id) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", fake_create_aio) manager = SandboxManager() await manager.restore_from_snapshot( @@ -281,8 +282,8 @@ class FakeSandbox: return FakeSandbox() fake_create_aio.aio = fake_create_aio - monkeypatch.setattr("src.sandbox.manager.modal.Image.from_id", fake_from_id) - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", fake_create_aio) + monkeypatch.setattr("src.sandbox.launch.modal.Image.from_id", fake_from_id) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", fake_create_aio) manager = SandboxManager() await manager.restore_from_snapshot( @@ -327,8 +328,8 @@ class FakeSandbox: return FakeSandbox() fake_create_aio.aio = fake_create_aio - monkeypatch.setattr("src.sandbox.manager.modal.Image.from_id", fake_from_id) - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", fake_create_aio) + monkeypatch.setattr("src.sandbox.launch.modal.Image.from_id", fake_from_id) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", fake_create_aio) manager = SandboxManager() @@ -369,8 +370,8 @@ def _fake_restore_setup(monkeypatch): class FakeImage: object_id = "img-123" - monkeypatch.setattr("src.sandbox.manager.modal.Image.from_id", lambda *a, **kw: FakeImage()) - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", _fake_sandbox_create(captured)) + monkeypatch.setattr("src.sandbox.launch.modal.Image.from_id", lambda *a, **kw: FakeImage()) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", _fake_sandbox_create(captured)) return captured @@ -466,7 +467,7 @@ class FakeSandbox: async def test_vcs_env_vars_default_github(monkeypatch): """SCM_PROVIDER unset → github.com defaults, no token in env.""" captured = {} - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", _fake_sandbox_create(captured)) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", _fake_sandbox_create(captured)) monkeypatch.delenv("SCM_PROVIDER", raising=False) manager = SandboxManager() @@ -488,7 +489,7 @@ async def test_vcs_env_vars_default_github(monkeypatch): async def test_vcs_env_vars_gitlab(monkeypatch): """SCM_PROVIDER=gitlab → gitlab.com + oauth2, no token in env.""" captured = {} - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", _fake_sandbox_create(captured)) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", _fake_sandbox_create(captured)) monkeypatch.setenv("SCM_PROVIDER", "gitlab") manager = SandboxManager() @@ -508,7 +509,7 @@ async def test_vcs_env_vars_gitlab(monkeypatch): async def test_vcs_env_vars_bitbucket(monkeypatch): """SCM_PROVIDER=bitbucket → bitbucket.org + x-token-auth, no token in env.""" captured = {} - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", _fake_sandbox_create(captured)) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", _fake_sandbox_create(captured)) monkeypatch.setenv("SCM_PROVIDER", "bitbucket") manager = SandboxManager() @@ -532,8 +533,8 @@ async def test_repo_image_boot_omits_fallback_tokens(monkeypatch): class FakeImage: object_id = "repo-img-1" - monkeypatch.setattr("src.sandbox.manager.modal.Image.from_id", lambda *a, **kw: FakeImage()) - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", _fake_sandbox_create(captured)) + monkeypatch.setattr("src.sandbox.launch.modal.Image.from_id", lambda *a, **kw: FakeImage()) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", _fake_sandbox_create(captured)) monkeypatch.delenv("SCM_PROVIDER", raising=False) manager = SandboxManager() @@ -561,8 +562,8 @@ async def test_repo_image_boot_preserves_user_github_cli_token(monkeypatch, toke class FakeImage: object_id = "repo-img-1" - monkeypatch.setattr("src.sandbox.manager.modal.Image.from_id", lambda *a, **kw: FakeImage()) - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", _fake_sandbox_create(captured)) + monkeypatch.setattr("src.sandbox.launch.modal.Image.from_id", lambda *a, **kw: FakeImage()) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", _fake_sandbox_create(captured)) monkeypatch.delenv("SCM_PROVIDER", raising=False) manager = SandboxManager() @@ -591,7 +592,7 @@ async def test_no_repo_sandbox_gets_provider_host_scoping(monkeypatch): fall back to github.com credential-helper behavior. """ captured = {} - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", _fake_sandbox_create(captured)) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", _fake_sandbox_create(captured)) monkeypatch.setenv("SCM_PROVIDER", "gitlab") manager = SandboxManager() @@ -611,8 +612,8 @@ async def test_restore_no_repo_gets_host_scoping_without_tokens(monkeypatch): class FakeImage: object_id = "img-123" - monkeypatch.setattr("src.sandbox.manager.modal.Image.from_id", lambda *a, **kw: FakeImage()) - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", _fake_sandbox_create(captured)) + monkeypatch.setattr("src.sandbox.launch.modal.Image.from_id", lambda *a, **kw: FakeImage()) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", _fake_sandbox_create(captured)) monkeypatch.setenv("SCM_PROVIDER", "bitbucket") manager = SandboxManager() @@ -649,8 +650,8 @@ async def test_restore_preserves_vcs_clone_token_for_legacy_snapshots(monkeypatc class FakeImage: object_id = "img-123" - monkeypatch.setattr("src.sandbox.manager.modal.Image.from_id", lambda *a, **kw: FakeImage()) - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", _fake_sandbox_create(captured)) + monkeypatch.setattr("src.sandbox.launch.modal.Image.from_id", lambda *a, **kw: FakeImage()) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", _fake_sandbox_create(captured)) monkeypatch.setenv("SCM_PROVIDER", "bitbucket") manager = SandboxManager() @@ -685,8 +686,8 @@ async def test_restore_github_includes_gh_cli_aliases(monkeypatch): class FakeImage: object_id = "img-123" - monkeypatch.setattr("src.sandbox.manager.modal.Image.from_id", lambda *a, **kw: FakeImage()) - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", _fake_sandbox_create(captured)) + monkeypatch.setattr("src.sandbox.launch.modal.Image.from_id", lambda *a, **kw: FakeImage()) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", _fake_sandbox_create(captured)) monkeypatch.delenv("SCM_PROVIDER", raising=False) manager = SandboxManager() @@ -720,8 +721,8 @@ async def test_no_repo_restore_omits_clone_token(monkeypatch): class FakeImage: object_id = "img-123" - monkeypatch.setattr("src.sandbox.manager.modal.Image.from_id", lambda *a, **kw: FakeImage()) - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", _fake_sandbox_create(captured)) + monkeypatch.setattr("src.sandbox.launch.modal.Image.from_id", lambda *a, **kw: FakeImage()) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", _fake_sandbox_create(captured)) monkeypatch.delenv("SCM_PROVIDER", raising=False) manager = SandboxManager() diff --git a/packages/modal-infra/tests/test_sandbox_launch.py b/packages/modal-infra/tests/test_sandbox_launch.py index 1fb73b3e32..5831ebbf18 100644 --- a/packages/modal-infra/tests/test_sandbox_launch.py +++ b/packages/modal-infra/tests/test_sandbox_launch.py @@ -6,6 +6,7 @@ from unittest.mock import AsyncMock, Mock import pytest +from modal.exception import NotFoundError from sandbox_runtime.constants import ( CODE_SERVER_PORT_ENV_VAR, @@ -13,9 +14,12 @@ EXPECTED_TUNNEL_PORTS_ENV_VAR, NOVNC_PORT_ENV_VAR, TTYD_PROXY_PORT_ENV_VAR, + TUNNEL_ENV_FILE_PATH, + TUNNEL_ENV_SANDBOX_ID_KEY, VNC_PASSWORD_ENV_VAR, ) -from sandbox_runtime.types import SessionConfig +from sandbox_runtime.types import SandboxStatus, SessionConfig +from src.sandbox.launch import SandboxLauncher from src.sandbox.launch_policy import ( DockerImageUnavailableError, InvalidDockerSettingsError, @@ -27,13 +31,25 @@ SandboxConfig, SandboxManager, ) +from src.sandbox.tunnels import SandboxTunnels, TunnelUrls def _fake_create(captured: dict): async def create_aio(*args, **kwargs): captured["command"] = args captured["kwargs"] = kwargs - return SimpleNamespace(object_id="modal-object-1", stdout=None) + return SimpleNamespace( + object_id="modal-object-1", + tunnels=Mock( + return_value={ + 9000: SimpleNamespace(url="https://code.example"), + 9001: SimpleNamespace(url="https://vnc.example"), + 9002: SimpleNamespace(url="https://terminal.example"), + 3000: SimpleNamespace(url="https://app.example"), + } + ), + filesystem=SimpleNamespace(write_text=SimpleNamespace(aio=AsyncMock())), + ) create_aio.aio = create_aio return create_aio @@ -50,27 +66,14 @@ async def test_launch_matrix_preserves_common_and_source_specific_behavior( "repo-image-1": object(), "snapshot-image-1": object(), } - monkeypatch.setattr("src.sandbox.manager.base_image", base_image) - monkeypatch.setattr("src.sandbox.manager.modal.Image.from_id", images.__getitem__) - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", _fake_create(captured)) + monkeypatch.setattr("src.sandbox.launch.base_image", base_image) + monkeypatch.setattr("src.sandbox.launch.modal.Image.from_id", images.__getitem__) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", _fake_create(captured)) monkeypatch.delenv("SCM_PROVIDER", raising=False) - resolve_tunnels = AsyncMock( - return_value=( - "https://code.example", - "https://vnc.example", - "https://terminal.example", - {3000: "https://app.example"}, - ) - ) - monkeypatch.setattr( - SandboxManager, - "_resolve_and_setup_tunnels", - resolve_tunnels, - ) monkeypatch.setattr( - SandboxManager, "_generate_code_server_password", staticmethod(lambda: "code-password") + SandboxLauncher, "_generate_code_server_password", staticmethod(lambda: "code-password") ) - monkeypatch.setattr(SandboxManager, "_generate_vnc_password", staticmethod(lambda: "vnc-pass")) + monkeypatch.setattr(SandboxLauncher, "_generate_vnc_password", staticmethod(lambda: "vnc-pass")) manager = SandboxManager() settings = { @@ -189,23 +192,17 @@ async def test_launch_matrix_preserves_common_and_source_specific_behavior( assert handle.vnc_password == "vnc-pass" assert handle.ttyd_url == "https://terminal.example" assert handle.tunnel_urls == {3000: "https://app.example"} - resolve_tunnels.assert_awaited_once_with( - handle.modal_sandbox, - "sandbox-1", - True, - True, - True, - [3000], - 9000, - 9001, - 9002, + handle.modal_sandbox.tunnels.assert_called_once_with() + handle.modal_sandbox.filesystem.write_text.aio.assert_awaited_once_with( + f"{TUNNEL_ENV_SANDBOX_ID_KEY}=sandbox-1\nTUNNEL_3000=https://app.example\n", + TUNNEL_ENV_FILE_PATH, ) @pytest.mark.asyncio async def test_repository_image_create_validates_repo_before_image_lookup(monkeypatch): from_id = Mock(side_effect=AssertionError("image lookup should not run")) - monkeypatch.setattr("src.sandbox.manager.modal.Image.from_id", from_id) + monkeypatch.setattr("src.sandbox.launch.modal.Image.from_id", from_id) with pytest.raises(ValueError, match="repo_owner and repo_name must be provided together"): await SandboxManager().create_sandbox( @@ -216,25 +213,147 @@ async def test_repository_image_create_validates_repo_before_image_lookup(monkey @pytest.mark.asyncio -async def test_repository_image_not_found_is_reported_explicitly(monkeypatch, fake_llm_secret): - from modal.exception import NotFoundError +@pytest.mark.parametrize("image_source", ["repository", "snapshot"]) +@pytest.mark.parametrize("failure_stage", ["lookup", "create"]) +@pytest.mark.parametrize("missing", [False, True]) +async def test_launch_preserves_image_error_classification( + monkeypatch, image_source, failure_stage, missing +): + error = NotFoundError("missing image") if missing else RuntimeError("transient failure") + from_id = Mock( + return_value=object(), + side_effect=error if failure_stage == "lookup" else None, + ) + create = AsyncMock(side_effect=error) + monkeypatch.setattr("src.sandbox.launch.modal.Image.from_id", from_id) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", SimpleNamespace(aio=create)) + expected_error = ( + RepositoryImageUnavailableError if image_source == "repository" and missing else type(error) + ) + + with pytest.raises(expected_error) as raised: + if image_source == "snapshot": + await SandboxManager().restore_from_snapshot( + snapshot_image_id="image-1", + session_config={"repo_owner": "acme", "repo_name": "repo"}, + ) + else: + await SandboxManager().create_sandbox( + SandboxConfig(repo_owner="acme", repo_name="repo", repo_image_id="image-1") + ) + + if expected_error is RepositoryImageUnavailableError: + assert raised.value.__cause__ is error + else: + assert raised.value is error + from_id.assert_called_once_with("image-1") + if failure_stage == "lookup": + create.assert_not_awaited() + else: + # A spawn error must not silently fall back to a different image or retry. + create.assert_awaited_once() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("missing", [False, True]) +async def test_base_image_spawn_errors_propagate_without_retry(monkeypatch, missing): + error = NotFoundError("missing image") if missing else RuntimeError("transient failure") + create = AsyncMock(side_effect=error) + from_id = Mock() + monkeypatch.setattr("src.sandbox.launch.modal.Image.from_id", from_id) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", SimpleNamespace(aio=create)) + + with pytest.raises(type(error)) as raised: + await SandboxManager().create_sandbox(SandboxConfig(repo_owner=None, repo_name=None)) + + assert raised.value is error + create.assert_awaited_once() + from_id.assert_not_called() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("image_source", ["base", "repository", "snapshot"]) +@pytest.mark.parametrize("failure", ["partial", "unavailable", "write"]) +async def test_launch_returns_handle_despite_tunnel_failures(monkeypatch, image_source, failure): + write_text = AsyncMock(side_effect=OSError("write failed") if failure == "write" else None) + sandbox = SimpleNamespace( + object_id="modal-object-1", + tunnels=Mock( + side_effect=( + [RuntimeError("unavailable")] * 3 + if failure == "unavailable" + else [ + {9000: SimpleNamespace(url="https://code.example")}, + RuntimeError("not ready"), + {3000: SimpleNamespace(url="https://app.example")}, + ] + ) + ), + filesystem=SimpleNamespace(write_text=SimpleNamespace(aio=write_text)), + ) + create = AsyncMock(return_value=sandbox) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", SimpleNamespace(aio=create)) + monkeypatch.setattr("src.sandbox.launch.modal.Image.from_id", lambda _: object()) + sleep = AsyncMock() + monkeypatch.setattr("src.sandbox.tunnels.asyncio.sleep", sleep) + common = { + "sandbox_id": "sandbox-partial", + "code_server_enabled": True, + "settings": {"codeServerPort": 9000, "tunnelPorts": [3000, 3001]}, + } + manager = SandboxManager() + + if image_source == "snapshot": + handle = await manager.restore_from_snapshot( + snapshot_image_id="image-1", + session_config={"repo_owner": "acme", "repo_name": "repo"}, + **common, + ) + else: + handle = await manager.create_sandbox( + SandboxConfig( + repo_owner="acme", + repo_name="repo", + repo_image_id="image-1" if image_source == "repository" else None, + **common, + ) + ) - monkeypatch.setattr("src.sandbox.manager.modal.Image.from_id", lambda _image_id: object()) + assert handle.status is SandboxStatus.WARMING + assert handle.modal_sandbox is sandbox + assert handle.modal_object_id == "modal-object-1" + assert handle.code_server_password == create.call_args.kwargs["env"]["CODE_SERVER_PASSWORD"] + assert create.call_args.kwargs["encrypted_ports"] == [9000, 3000, 3001] + assert sandbox.tunnels.call_count == 3 + assert [call.args for call in sleep.await_args_list] == [(1.0,), (2.0,)] + create.assert_awaited_once() + if failure == "unavailable": + assert handle.code_server_url is None + assert handle.tunnel_urls is None + write_text.assert_not_awaited() + else: + assert handle.code_server_url == "https://code.example" + assert handle.tunnel_urls == {3000: "https://app.example"} + write_text.assert_awaited_once_with( + f"{TUNNEL_ENV_SANDBOX_ID_KEY}=sandbox-partial\nTUNNEL_3000=https://app.example\n", + TUNNEL_ENV_FILE_PATH, + ) + + +@pytest.mark.asyncio +async def test_repository_image_not_found_is_reported_explicitly(monkeypatch, fake_llm_secret): + monkeypatch.setattr("src.sandbox.launch.modal.Image.from_id", lambda _image_id: object()) async def create_aio(*_args, **_kwargs): fake_llm_secret[0].hydrate.aio.assert_awaited_once_with() raise NotFoundError("image not found") create = SimpleNamespace(aio=AsyncMock(side_effect=create_aio)) - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", create) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", create) with pytest.raises(RepositoryImageUnavailableError) as exc_info: await SandboxManager().create_sandbox( - SandboxConfig( - repo_owner="acme", - repo_name="repo", - repo_image_id="repo-image-missing", - ) + SandboxConfig(repo_owner="acme", repo_name="repo", repo_image_id="image-1") ) assert isinstance(exc_info.value.__cause__, NotFoundError) @@ -243,17 +362,15 @@ async def create_aio(*_args, **_kwargs): @pytest.mark.asyncio async def test_missing_secret_does_not_mark_repository_image_unavailable(monkeypatch): - from modal.exception import NotFoundError - create = SimpleNamespace(aio=AsyncMock()) - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", create) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", create) def missing_secret(_name, **_kwargs): secret = Mock() secret.hydrate.aio = AsyncMock(side_effect=NotFoundError("secret not found")) return secret - monkeypatch.setattr("src.sandbox.manager.modal.Secret.from_name", missing_secret) + monkeypatch.setattr("src.sandbox.launch.modal.Secret.from_name", missing_secret) with pytest.raises(NotFoundError, match="secret not found"): await SandboxManager().create_sandbox( @@ -265,10 +382,8 @@ def missing_secret(_name, **_kwargs): @pytest.mark.asyncio async def test_base_image_not_found_is_not_classified_as_repository_image(monkeypatch): - from modal.exception import NotFoundError - create = SimpleNamespace(aio=AsyncMock(side_effect=NotFoundError("base image not found"))) - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", create) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", create) with pytest.raises(NotFoundError, match="base image not found"): await SandboxManager().create_sandbox(SandboxConfig(repo_owner="acme", repo_name="repo")) @@ -282,13 +397,12 @@ async def test_base_image_not_found_is_not_classified_as_repository_image(monkey def _docker_manager(monkeypatch) -> tuple[SandboxManager, dict, object]: captured: dict = {} docker_image = object() - monkeypatch.setattr("src.sandbox.manager.base_image", object()) + monkeypatch.setattr("src.sandbox.launch.base_image", object()) monkeypatch.setattr("src.images.base.docker_image", docker_image) - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", _fake_create(captured)) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", _fake_create(captured)) monkeypatch.setattr( - SandboxManager, - "_resolve_and_setup_tunnels", - AsyncMock(return_value=(None, None, None, {})), + "src.sandbox.tunnels.SandboxTunnels.resolve", + AsyncMock(return_value=TunnelUrls()), ) return SandboxManager(), captured, docker_image @@ -311,8 +425,6 @@ def _docker_config(**overrides) -> SandboxConfig: def _not_found(*_args, **_kwargs): - from modal.exception import NotFoundError - raise NotFoundError("no sandbox") @@ -321,9 +433,9 @@ def _not_found(*_args, **_kwargs): async def test_docker_launch_selects_vm_runtime_and_named_allocation(monkeypatch, image_source): manager, captured, docker_image = _docker_manager(monkeypatch) artifact = object() - monkeypatch.setattr("src.sandbox.manager.modal.Image.from_id", lambda _id: artifact) + monkeypatch.setattr("src.sandbox.launch.modal.Image.from_id", lambda _id: artifact) monkeypatch.setattr( - "src.sandbox.manager.modal.Sandbox.from_name", + "src.sandbox.launch.modal.Sandbox.from_name", SimpleNamespace(aio=AsyncMock(side_effect=_not_found)), ) @@ -350,7 +462,6 @@ async def test_docker_launch_selects_vm_runtime_and_named_allocation(monkeypatch assert kwargs["memory"] == 4096 assert kwargs["name"] == docker_allocation_name("session-1") assert kwargs["tags"] == docker_allocation_tags("session-1", "sandbox-acme-repo-1700000000000") - # The trusted signal wins over any user-supplied value. assert kwargs["env"][DOCKER_ENABLED_ENV_VAR] == "true" assert handle.sandbox_backend == "modal-vm" @@ -359,9 +470,9 @@ async def test_docker_launch_selects_vm_runtime_and_named_allocation(monkeypatch @pytest.mark.parametrize("image_source", ["base", "snapshot"]) async def test_expired_vm_launch_cannot_materialize_after_lookup(monkeypatch, image_source): manager, captured, _ = _docker_manager(monkeypatch) - monkeypatch.setattr("src.sandbox.manager.modal.Image.from_id", lambda _id: object()) + monkeypatch.setattr("src.sandbox.launch.modal.Image.from_id", lambda _id: object()) monkeypatch.setattr( - "src.sandbox.manager.modal.Sandbox.from_name", + "src.sandbox.launch.modal.Sandbox.from_name", SimpleNamespace(aio=AsyncMock(side_effect=_not_found)), ) if image_source == "base": @@ -409,7 +520,7 @@ async def test_docker_launch_adopts_an_existing_owned_allocation(monkeypatch): existing.get_tags.aio = existing.get_tags from_name = AsyncMock(return_value=existing) monkeypatch.setattr( - "src.sandbox.manager.modal.Sandbox.from_name", SimpleNamespace(aio=from_name) + "src.sandbox.launch.modal.Sandbox.from_name", SimpleNamespace(aio=from_name) ) handle = await manager.create_sandbox(_docker_config()) @@ -425,19 +536,19 @@ async def test_docker_launch_adopts_an_existing_owned_allocation(monkeypatch): async def test_docker_retry_returns_the_original_access_credentials( monkeypatch, create_race, image_source ): - from modal.exception import AlreadyExistsError, NotFoundError + from modal.exception import AlreadyExistsError manager, captured, _ = _docker_manager(monkeypatch) - monkeypatch.setattr("src.sandbox.manager.modal.Image.from_id", lambda _id: object()) + monkeypatch.setattr("src.sandbox.launch.modal.Image.from_id", lambda _id: object()) monkeypatch.setattr( - SandboxManager, "_generate_code_server_password", Mock(side_effect=["original", "new"]) + SandboxLauncher, "_generate_code_server_password", Mock(side_effect=["original", "new"]) ) monkeypatch.setattr( - SandboxManager, "_generate_vnc_password", Mock(side_effect=["old-vnc", "new-vnc"]) + SandboxLauncher, "_generate_vnc_password", Mock(side_effect=["old-vnc", "new-vnc"]) ) from_name = AsyncMock(side_effect=NotFoundError("not created")) monkeypatch.setattr( - "src.sandbox.manager.modal.Sandbox.from_name", SimpleNamespace(aio=from_name) + "src.sandbox.launch.modal.Sandbox.from_name", SimpleNamespace(aio=from_name) ) async def launch(): @@ -470,7 +581,7 @@ async def launch(): ) from_name.side_effect = [NotFoundError("racing"), existing] if create_race else [existing] create = AsyncMock(side_effect=AlreadyExistsError("already created")) - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", SimpleNamespace(aio=create)) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", SimpleNamespace(aio=create)) adopted = await launch() @@ -503,7 +614,7 @@ async def test_docker_adoption_fails_if_original_credentials_cannot_be_recovered exec=SimpleNamespace(aio=AsyncMock(return_value=process)), ) monkeypatch.setattr( - "src.sandbox.manager.modal.Sandbox.from_name", + "src.sandbox.launch.modal.Sandbox.from_name", SimpleNamespace(aio=AsyncMock(return_value=existing)), ) @@ -511,7 +622,7 @@ async def test_docker_adoption_fails_if_original_credentials_cannot_be_recovered await manager.create_sandbox(_docker_config(code_server_enabled=True)) assert "kwargs" not in captured - manager._resolve_and_setup_tunnels.assert_not_awaited() + SandboxTunnels.resolve.assert_not_awaited() @pytest.mark.asyncio @@ -523,7 +634,7 @@ async def test_docker_launch_refuses_a_same_named_allocation_it_does_not_own(mon ) foreign.get_tags.aio = foreign.get_tags monkeypatch.setattr( - "src.sandbox.manager.modal.Sandbox.from_name", + "src.sandbox.launch.modal.Sandbox.from_name", SimpleNamespace(aio=AsyncMock(return_value=foreign)), ) @@ -553,7 +664,7 @@ async def from_name(_app, name): _not_found() monkeypatch.setattr( - "src.sandbox.manager.modal.Sandbox.from_name", SimpleNamespace(aio=from_name) + "src.sandbox.launch.modal.Sandbox.from_name", SimpleNamespace(aio=from_name) ) await manager.create_sandbox( @@ -563,7 +674,6 @@ async def from_name(_app, name): prior.terminate.assert_awaited_once_with(wait=True) assert captured["kwargs"]["name"] == docker_allocation_name("session-1") - # A prior allocation with foreign tags is left alone. prior.terminate.reset_mock() prior.get_tags = AsyncMock(return_value={"openinspect_kind": "other"}) prior.get_tags.aio = prior.get_tags @@ -576,9 +686,10 @@ async def from_name(_app, name): @pytest.mark.asyncio async def test_late_predecessor_cannot_materialize_beside_successor(monkeypatch): - from modal.exception import AlreadyExistsError, NotFoundError + from modal.exception import AlreadyExistsError - manager, _, _ = _docker_manager(monkeypatch) + _, _, _ = _docker_manager(monkeypatch) + launcher = SandboxLauncher() predecessor_name = docker_allocation_name("session-1") predecessor = SimpleNamespace( object_id="late-predecessor", @@ -589,17 +700,16 @@ async def test_late_predecessor_cannot_materialize_beside_successor(monkeypatch) lookup = AsyncMock( side_effect=[NotFoundError("still creating"), NotFoundError("still creating"), predecessor] ) - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.from_name", SimpleNamespace(aio=lookup)) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.from_name", SimpleNamespace(aio=lookup)) async def create(kwargs, *, repository_image): - # Provider-side naming wins the race after both client lookups missed it. if kwargs["name"] == predecessor_name: raise AlreadyExistsError("predecessor won the name") return SimpleNamespace(object_id="duplicate-successor") - monkeypatch.setattr("src.sandbox.manager._create_sandbox", create) + monkeypatch.setattr("src.sandbox.launch._create_sandbox", create) with pytest.raises(RuntimeError, match="ownership mismatch"): - await manager._launch_docker_sandbox( + await launcher._launch_docker_sandbox( session_id="session-1", sandbox_id="successor", retire_sandbox_id="prior", @@ -631,10 +741,8 @@ async def terminate(*, wait=False): ), terminate=SimpleNamespace(aio=terminate), ) - from modal.exception import NotFoundError - monkeypatch.setattr( - "src.sandbox.manager.modal.Sandbox.from_name", + "src.sandbox.launch.modal.Sandbox.from_name", SimpleNamespace(aio=AsyncMock(side_effect=[prior, NotFoundError("no successor")])), ) launch = asyncio.create_task( @@ -656,3 +764,50 @@ async def terminate(*, wait=False): finally: launch.cancel() await asyncio.gather(launch, return_exceptions=True) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("restore", [False, True], ids=["create", "restore"]) +@pytest.mark.parametrize( + "ports, expected", + [ + ([True, False], []), + ([True, False, 0, -1, 65536, "3000", 3.5, None, 1, 3000, 65535], [1, 3000, 65535]), + ([True] * 10 + [3000], [3000]), + ], + ids=["booleans-only", "mixed-with-boundary-ports", "booleans-do-not-consume-limit"], +) +async def test_launch_rejects_boolean_tunnel_ports(monkeypatch, restore, ports, expected): + """Invalid extras never reach Modal or the runtime's expected-port list.""" + urls = {port: f"https://port-{port}.example" for port in expected} + sandbox = SimpleNamespace( + object_id="modal-ports", + tunnels=Mock(return_value={port: SimpleNamespace(url=url) for port, url in urls.items()}), + filesystem=SimpleNamespace(write_text=SimpleNamespace(aio=AsyncMock())), + ) + create = AsyncMock(return_value=sandbox) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", SimpleNamespace(aio=create)) + monkeypatch.setattr("src.sandbox.launch.modal.Image.from_id", lambda _: object()) + manager = SandboxManager() + settings = {"tunnelPorts": ports} + + if restore: + handle = await manager.restore_from_snapshot( + snapshot_image_id="image-1", + session_config={"repo_owner": "acme", "repo_name": "repo"}, + settings=settings, + ) + else: + handle = await manager.create_sandbox( + SandboxConfig(repo_owner="acme", repo_name="repo", settings=settings) + ) + + kwargs = create.call_args.kwargs + assert kwargs.get("encrypted_ports", []) == expected + assert all(type(port) is int for port in kwargs.get("encrypted_ports", [])) + assert kwargs["env"].get(EXPECTED_TUNNEL_PORTS_ENV_VAR) == ( + ",".join(str(port) for port in expected) if expected else None + ) + assert handle.tunnel_urls == (urls or None) + if not expected: + sandbox.tunnels.assert_not_called() diff --git a/packages/modal-infra/tests/test_sandbox_resources.py b/packages/modal-infra/tests/test_sandbox_resources.py index cb8d7a1b90..2505511766 100644 --- a/packages/modal-infra/tests/test_sandbox_resources.py +++ b/packages/modal-infra/tests/test_sandbox_resources.py @@ -6,6 +6,7 @@ from src.sandbox.launch_policy import launch_kwargs, parse_launch from src.sandbox.manager import SandboxConfig, SandboxManager +from src.sandbox.tunnels import SandboxTunnels, TunnelUrls class TestResourceKwargs: @@ -47,11 +48,11 @@ class TestCreateSandboxResources: @pytest.mark.asyncio async def test_create_sandbox_passes_cpu_and_memory(self, monkeypatch): captured: dict = {} - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", _fake_create(captured)) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", _fake_create(captured)) monkeypatch.setattr( - SandboxManager, - "_resolve_and_setup_tunnels", - AsyncMock(return_value=(None, None, None, None)), + SandboxTunnels, + "resolve", + AsyncMock(return_value=TunnelUrls(None, None, None, None)), ) manager = SandboxManager() @@ -74,13 +75,13 @@ class FakeImage: object_id = "img-1" monkeypatch.setattr( - "src.sandbox.manager.modal.Image.from_id", lambda *_a, **_kw: FakeImage() + "src.sandbox.launch.modal.Image.from_id", lambda *_a, **_kw: FakeImage() ) - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", _fake_create(captured)) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", _fake_create(captured)) monkeypatch.setattr( - SandboxManager, - "_resolve_and_setup_tunnels", - AsyncMock(return_value=(None, None, None, None)), + SandboxTunnels, + "resolve", + AsyncMock(return_value=TunnelUrls(None, None, None, None)), ) manager = SandboxManager() diff --git a/packages/modal-infra/tests/test_ttyd.py b/packages/modal-infra/tests/test_ttyd.py index 015cb456e5..3ddd6ae978 100644 --- a/packages/modal-infra/tests/test_ttyd.py +++ b/packages/modal-infra/tests/test_ttyd.py @@ -4,74 +4,38 @@ import pytest -from sandbox_runtime.constants import NOVNC_PORT, TTYD_PORT +from sandbox_runtime.constants import ( + EXPECTED_TUNNEL_PORTS_ENV_VAR, + NOVNC_PORT, + TTYD_PORT, +) from src.sandbox.manager import ( CODE_SERVER_PORT, TTYD_PROXY_PORT, SandboxConfig, SandboxManager, ) +from src.sandbox.tunnels import SandboxTunnels, TunnelUrls -class TestCollectExposedPortsTerminal: - """_collect_exposed_ports with terminal_enabled flag.""" +@pytest.mark.parametrize("code_server", [False, True]) +@pytest.mark.parametrize("terminal", [False, True]) +def test_terminal_port_ownership(code_server, terminal): + tunnels = SandboxTunnels( + code_server_enabled=code_server, + settings={"terminalEnabled": terminal}, + ) + assert (TTYD_PROXY_PORT in tunnels.exposed_ports) is terminal + assert (CODE_SERVER_PORT in tunnels.exposed_ports) is code_server + assert TTYD_PORT not in tunnels.exposed_ports - def test_terminal_enabled_includes_proxy_port(self): - exposed, _extra = SandboxManager._collect_exposed_ports( - code_server_enabled=False, - vnc_enabled=False, - terminal_enabled=True, - settings=None, - code_server_port=CODE_SERVER_PORT, - novnc_port=NOVNC_PORT, - ttyd_proxy_port=TTYD_PROXY_PORT, - ) - assert TTYD_PROXY_PORT in exposed - # ttyd raw port should NOT be exposed (only the proxy port) - assert TTYD_PORT not in exposed - def test_terminal_disabled_excludes_proxy_port(self): - exposed, _extra = SandboxManager._collect_exposed_ports( - code_server_enabled=False, - vnc_enabled=False, - terminal_enabled=False, - settings=None, - code_server_port=CODE_SERVER_PORT, - novnc_port=NOVNC_PORT, - ttyd_proxy_port=TTYD_PROXY_PORT, - ) - assert TTYD_PROXY_PORT not in exposed - - def test_terminal_and_code_server_both_enabled(self): - exposed, _extra = SandboxManager._collect_exposed_ports( - code_server_enabled=True, - vnc_enabled=False, - terminal_enabled=True, - settings=None, - code_server_port=CODE_SERVER_PORT, - novnc_port=NOVNC_PORT, - ttyd_proxy_port=TTYD_PROXY_PORT, - ) - assert CODE_SERVER_PORT in exposed - assert TTYD_PROXY_PORT in exposed - - def test_terminal_port_deduped_from_tunnel_ports(self): - """If user explicitly lists TTYD_PROXY_PORT in tunnelPorts, it should not duplicate.""" - settings = {"tunnelPorts": [TTYD_PROXY_PORT, 3000]} - exposed, extra = SandboxManager._collect_exposed_ports( - code_server_enabled=False, - vnc_enabled=False, - terminal_enabled=True, - settings=settings, - code_server_port=CODE_SERVER_PORT, - novnc_port=NOVNC_PORT, - ttyd_proxy_port=TTYD_PROXY_PORT, - ) - assert exposed.count(TTYD_PROXY_PORT) == 1 - assert 3000 in exposed - # TTYD_PROXY_PORT should not be in extra (reserved) - assert TTYD_PROXY_PORT not in extra - assert 3000 in extra +def test_terminal_port_is_not_duplicated_by_extra_tunnel(): + tunnels = SandboxTunnels( + settings={"terminalEnabled": True, "tunnelPorts": [TTYD_PROXY_PORT, 3000]} + ) + assert tunnels.exposed_ports == [TTYD_PROXY_PORT, 3000] + assert tunnels.environment[EXPECTED_TUNNEL_PORTS_ENV_VAR] == "3000" class TestResolveTunnelsTerminal: @@ -85,17 +49,17 @@ async def test_returns_ttyd_url_when_terminal_enabled(self): sandbox = MagicMock() sandbox.tunnels.return_value = {TTYD_PROXY_PORT: tunnel} - cs_url, vnc_url, ttyd_url, extra = await SandboxManager._resolve_and_setup_tunnels( - sandbox, - "sb-123", + cs_url, vnc_url, ttyd_url, extra = await SandboxTunnels( code_server_enabled=False, vnc_enabled=False, - terminal_enabled=True, - extra_ports=[], - code_server_port=CODE_SERVER_PORT, - novnc_port=NOVNC_PORT, - ttyd_proxy_port=TTYD_PROXY_PORT, - ) + settings={ + "terminalEnabled": True, + "tunnelPorts": [], + "codeServerPort": CODE_SERVER_PORT, + "vncPort": NOVNC_PORT, + "terminalPort": TTYD_PROXY_PORT, + }, + ).resolve(sandbox, "sb-123") assert cs_url is None assert vnc_url is None assert ttyd_url == "https://ttyd.example.com" @@ -104,17 +68,17 @@ async def test_returns_ttyd_url_when_terminal_enabled(self): @pytest.mark.asyncio async def test_returns_none_when_terminal_disabled(self): sandbox = MagicMock() - cs_url, vnc_url, ttyd_url, extra = await SandboxManager._resolve_and_setup_tunnels( - sandbox, - "sb-123", + cs_url, vnc_url, ttyd_url, extra = await SandboxTunnels( code_server_enabled=False, vnc_enabled=False, - terminal_enabled=False, - extra_ports=[], - code_server_port=CODE_SERVER_PORT, - novnc_port=NOVNC_PORT, - ttyd_proxy_port=TTYD_PROXY_PORT, - ) + settings={ + "terminalEnabled": False, + "tunnelPorts": [], + "codeServerPort": CODE_SERVER_PORT, + "vncPort": NOVNC_PORT, + "terminalPort": TTYD_PROXY_PORT, + }, + ).resolve(sandbox, "sb-123") assert cs_url is None assert vnc_url is None assert ttyd_url is None @@ -133,17 +97,17 @@ async def test_both_code_server_and_terminal(self): TTYD_PROXY_PORT: ttyd_tunnel, } - cs_url, vnc_url, ttyd_url, extra = await SandboxManager._resolve_and_setup_tunnels( - sandbox, - "sb-123", + cs_url, vnc_url, ttyd_url, extra = await SandboxTunnels( code_server_enabled=True, vnc_enabled=False, - terminal_enabled=True, - extra_ports=[], - code_server_port=CODE_SERVER_PORT, - novnc_port=NOVNC_PORT, - ttyd_proxy_port=TTYD_PROXY_PORT, - ) + settings={ + "terminalEnabled": True, + "tunnelPorts": [], + "codeServerPort": CODE_SERVER_PORT, + "vncPort": NOVNC_PORT, + "terminalPort": TTYD_PROXY_PORT, + }, + ).resolve(sandbox, "sb-123") assert cs_url == "https://cs.example.com" assert vnc_url is None assert ttyd_url == "https://ttyd.example.com" @@ -169,12 +133,12 @@ class FakeSandbox: fake_create = MagicMock() fake_create.aio = fake_create_aio - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", fake_create) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", fake_create) monkeypatch.setattr( - SandboxManager, - "_resolve_and_setup_tunnels", - AsyncMock(return_value=(None, None, "https://ttyd.example.com", None)), + SandboxTunnels, + "resolve", + AsyncMock(return_value=TunnelUrls(None, None, "https://ttyd.example.com", None)), ) manager = SandboxManager() @@ -209,10 +173,10 @@ class FakeSandbox: fake_create = MagicMock() fake_create.aio = fake_create_aio - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", fake_create) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", fake_create) - tunnel_mock = AsyncMock(return_value=(None, None, None, None)) - monkeypatch.setattr(SandboxManager, "_resolve_and_setup_tunnels", tunnel_mock) + tunnel_mock = AsyncMock(return_value=TunnelUrls(None, None, None, None)) + monkeypatch.setattr(SandboxTunnels, "resolve", tunnel_mock) manager = SandboxManager() config = SandboxConfig( @@ -255,12 +219,14 @@ class FakeSandbox: fake_create = MagicMock() fake_create.aio = fake_create_aio - monkeypatch.setattr("src.sandbox.manager.modal.Image.from_id", fake_from_id) - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", fake_create) + monkeypatch.setattr("src.sandbox.launch.modal.Image.from_id", fake_from_id) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", fake_create) monkeypatch.setattr( - SandboxManager, - "_resolve_and_setup_tunnels", - AsyncMock(return_value=(None, None, "https://ttyd-restored.example.com", None)), + SandboxTunnels, + "resolve", + AsyncMock( + return_value=TunnelUrls(None, None, "https://ttyd-restored.example.com", None) + ), ) manager = SandboxManager() @@ -305,10 +271,10 @@ class FakeSandbox: fake_create = MagicMock() fake_create.aio = fake_create_aio - monkeypatch.setattr("src.sandbox.manager.modal.Image.from_id", fake_from_id) - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", fake_create) - tunnel_mock = AsyncMock(return_value=(None, None, None, None)) - monkeypatch.setattr(SandboxManager, "_resolve_and_setup_tunnels", tunnel_mock) + monkeypatch.setattr("src.sandbox.launch.modal.Image.from_id", fake_from_id) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", fake_create) + tunnel_mock = AsyncMock(return_value=TunnelUrls(None, None, None, None)) + monkeypatch.setattr(SandboxTunnels, "resolve", tunnel_mock) manager = SandboxManager() handle = await manager.restore_from_snapshot( diff --git a/packages/modal-infra/tests/test_tunnel_ports.py b/packages/modal-infra/tests/test_tunnel_ports.py index 68e51ab939..ead6c6b51a 100644 --- a/packages/modal-infra/tests/test_tunnel_ports.py +++ b/packages/modal-infra/tests/test_tunnel_ports.py @@ -14,6 +14,7 @@ TUNNEL_ENV_SANDBOX_ID_KEY, ) from src.sandbox.manager import CODE_SERVER_PORT, SandboxConfig, SandboxManager +from src.sandbox.tunnels import SandboxTunnels, TunnelUrls def _mock_sandbox_with_filesystem() -> tuple[MagicMock, AsyncMock]: @@ -27,7 +28,7 @@ def _mock_sandbox_with_filesystem() -> tuple[MagicMock, AsyncMock]: class TestResolveTunnels: - """SandboxManager._resolve_tunnels tests.""" + """SandboxTunnels._resolve_tunnels tests.""" @pytest.mark.asyncio async def test_resolves_all_ports(self): @@ -39,7 +40,7 @@ async def test_resolves_all_ports(self): sandbox = MagicMock() sandbox.tunnels.return_value = {3000: tunnel_3000, 3001: tunnel_3001} - result = await SandboxManager._resolve_tunnels(sandbox, "sb-1", [3000, 3001]) + result = await SandboxTunnels._resolve_tunnels(sandbox, "sb-1", [3000, 3001]) assert result == { 3000: "https://tunnel-3000.example.com", 3001: "https://tunnel-3001.example.com", @@ -53,9 +54,9 @@ async def test_returns_partial_on_missing_port(self): sandbox = MagicMock() sandbox.tunnels.return_value = {3000: tunnel_3000} - with patch("src.sandbox.manager.asyncio.sleep", new_callable=AsyncMock): - result = await SandboxManager._resolve_tunnels( - sandbox, "sb-1", [3000, 3001], retries=2, backoff=0.0 + with patch("src.sandbox.tunnels.asyncio.sleep", new_callable=AsyncMock): + result = await SandboxTunnels._resolve_tunnels( + sandbox, "sb-1", [3000, 3001], retries=2, backoff_seconds=0.0 ) assert result == {3000: "https://tunnel-3000.example.com"} @@ -64,9 +65,9 @@ async def test_returns_empty_on_exception_after_retries(self): sandbox = MagicMock() sandbox.tunnels.side_effect = Exception("tunnel unavailable") - with patch("src.sandbox.manager.asyncio.sleep", new_callable=AsyncMock): - result = await SandboxManager._resolve_tunnels( - sandbox, "sb-1", [3000], retries=3, backoff=0.0 + with patch("src.sandbox.tunnels.asyncio.sleep", new_callable=AsyncMock): + result = await SandboxTunnels._resolve_tunnels( + sandbox, "sb-1", [3000], retries=3, backoff_seconds=0.0 ) assert result == {} @@ -83,9 +84,9 @@ async def test_retries_on_partial_resolution(self): {3000: tunnel_3000, 3001: tunnel_3001}, ] - with patch("src.sandbox.manager.asyncio.sleep", new_callable=AsyncMock): - result = await SandboxManager._resolve_tunnels( - sandbox, "sb-1", [3000, 3001], retries=3, backoff=0.0 + with patch("src.sandbox.tunnels.asyncio.sleep", new_callable=AsyncMock): + result = await SandboxTunnels._resolve_tunnels( + sandbox, "sb-1", [3000, 3001], retries=3, backoff_seconds=0.0 ) assert result == { 3000: "https://tunnel-3000.example.com", @@ -100,17 +101,17 @@ class TestResolveAndSetupTunnels: @pytest.mark.asyncio async def test_returns_none_none_none_for_no_ports(self): sandbox = MagicMock() - cs_url, vnc_url, ttyd_url, extra = await SandboxManager._resolve_and_setup_tunnels( - sandbox, - "sb-1", - False, - False, - False, - [], - code_server_port=CODE_SERVER_PORT, - novnc_port=NOVNC_PORT, - ttyd_proxy_port=TTYD_PROXY_PORT, - ) + cs_url, vnc_url, ttyd_url, extra = await SandboxTunnels( + code_server_enabled=False, + vnc_enabled=False, + settings={ + "terminalEnabled": False, + "tunnelPorts": [], + "codeServerPort": CODE_SERVER_PORT, + "vncPort": NOVNC_PORT, + "terminalPort": TTYD_PROXY_PORT, + }, + ).resolve(sandbox, "sb-1") assert cs_url is None assert vnc_url is None assert ttyd_url is None @@ -122,22 +123,22 @@ async def test_resolves_extra_ports(self): sandbox, _write_text = _mock_sandbox_with_filesystem() with patch.object( - SandboxManager, + SandboxTunnels, "_resolve_tunnels", new_callable=AsyncMock, return_value=tunnel_urls, ): - cs_url, vnc_url, ttyd_url, extra = await SandboxManager._resolve_and_setup_tunnels( - sandbox, - "sb-1", - False, - False, - False, - [3000], - code_server_port=CODE_SERVER_PORT, - novnc_port=NOVNC_PORT, - ttyd_proxy_port=TTYD_PROXY_PORT, - ) + cs_url, vnc_url, ttyd_url, extra = await SandboxTunnels( + code_server_enabled=False, + vnc_enabled=False, + settings={ + "terminalEnabled": False, + "tunnelPorts": [3000], + "codeServerPort": CODE_SERVER_PORT, + "vncPort": NOVNC_PORT, + "terminalPort": TTYD_PROXY_PORT, + }, + ).resolve(sandbox, "sb-1") assert cs_url is None assert vnc_url is None @@ -154,22 +155,22 @@ async def test_splits_code_server_from_extra_ports(self): sandbox, _write_text = _mock_sandbox_with_filesystem() with patch.object( - SandboxManager, + SandboxTunnels, "_resolve_tunnels", new_callable=AsyncMock, return_value=resolved, ): - cs_url, vnc_url, ttyd_url, extra = await SandboxManager._resolve_and_setup_tunnels( - sandbox, - "sb-1", - True, - False, - False, - [3000], - code_server_port=CODE_SERVER_PORT, - novnc_port=NOVNC_PORT, - ttyd_proxy_port=TTYD_PROXY_PORT, - ) + cs_url, vnc_url, ttyd_url, extra = await SandboxTunnels( + code_server_enabled=True, + vnc_enabled=False, + settings={ + "terminalEnabled": False, + "tunnelPorts": [3000], + "codeServerPort": CODE_SERVER_PORT, + "vncPort": NOVNC_PORT, + "terminalPort": TTYD_PROXY_PORT, + }, + ).resolve(sandbox, "sb-1") assert cs_url == "https://cs.example.com" assert vnc_url is None @@ -183,22 +184,22 @@ async def test_keeps_code_server_port_tunnel_when_code_server_disabled(self): sandbox, _write_text = _mock_sandbox_with_filesystem() with patch.object( - SandboxManager, + SandboxTunnels, "_resolve_tunnels", new_callable=AsyncMock, return_value=resolved, ): - cs_url, vnc_url, ttyd_url, extra = await SandboxManager._resolve_and_setup_tunnels( - sandbox, - "sb-1", - False, - False, - False, - [CODE_SERVER_PORT], - code_server_port=CODE_SERVER_PORT, - novnc_port=NOVNC_PORT, - ttyd_proxy_port=TTYD_PROXY_PORT, - ) + cs_url, vnc_url, ttyd_url, extra = await SandboxTunnels( + code_server_enabled=False, + vnc_enabled=False, + settings={ + "terminalEnabled": False, + "tunnelPorts": [CODE_SERVER_PORT], + "codeServerPort": CODE_SERVER_PORT, + "vncPort": NOVNC_PORT, + "terminalPort": TTYD_PROXY_PORT, + }, + ).resolve(sandbox, "sb-1") assert cs_url is None assert vnc_url is None @@ -215,35 +216,35 @@ async def test_splits_custom_code_server_port_from_user_tunnel(self): sandbox, _write_text = _mock_sandbox_with_filesystem() with patch.object( - SandboxManager, + SandboxTunnels, "_resolve_tunnels", new_callable=AsyncMock, return_value=resolved, ): - cs_url, _vnc_url, _ttyd_url, extra = await SandboxManager._resolve_and_setup_tunnels( - sandbox, - "sb-1", - True, - False, - False, - [CODE_SERVER_PORT], - code_server_port=8081, - novnc_port=NOVNC_PORT, - ttyd_proxy_port=TTYD_PROXY_PORT, - ) + cs_url, _vnc_url, _ttyd_url, extra = await SandboxTunnels( + code_server_enabled=True, + vnc_enabled=False, + settings={ + "terminalEnabled": False, + "tunnelPorts": [CODE_SERVER_PORT], + "codeServerPort": 8081, + "vncPort": NOVNC_PORT, + "terminalPort": TTYD_PROXY_PORT, + }, + ).resolve(sandbox, "sb-1") assert cs_url == "https://cs.example.com" assert extra == {CODE_SERVER_PORT: "https://my-app.example.com"} class TestWriteTunnelEnvFile: - """SandboxManager._write_tunnel_env_file tests.""" + """SandboxTunnels._write_tunnel_env_file tests.""" @pytest.mark.asyncio async def test_writes_dotenv_format_to_expected_path(self): sandbox, write_text = _mock_sandbox_with_filesystem() - await SandboxManager._write_tunnel_env_file( + await SandboxTunnels._write_tunnel_env_file( sandbox, "sb-1", { @@ -267,8 +268,8 @@ async def test_write_failure_does_not_raise(self): sandbox, write_text = _mock_sandbox_with_filesystem() write_text.side_effect = Exception("write failed") - with patch("src.sandbox.manager.log") as mock_log: - await SandboxManager._write_tunnel_env_file( + with patch("src.sandbox.tunnels.log") as mock_log: + await SandboxTunnels._write_tunnel_env_file( sandbox, "sb-1", {3000: "https://tunnel-3000.example.com"} ) @@ -285,22 +286,22 @@ async def test_writes_file_when_extra_urls_present(self): tunnel_urls = {3000: "https://tunnel-3000.example.com"} with patch.object( - SandboxManager, + SandboxTunnels, "_resolve_tunnels", new_callable=AsyncMock, return_value=tunnel_urls, ): - await SandboxManager._resolve_and_setup_tunnels( - sandbox, - "sb-1", - False, - False, - False, - [3000], - code_server_port=CODE_SERVER_PORT, - novnc_port=NOVNC_PORT, - ttyd_proxy_port=TTYD_PROXY_PORT, - ) + await SandboxTunnels( + code_server_enabled=False, + vnc_enabled=False, + settings={ + "terminalEnabled": False, + "tunnelPorts": [3000], + "codeServerPort": CODE_SERVER_PORT, + "vncPort": NOVNC_PORT, + "terminalPort": TTYD_PROXY_PORT, + }, + ).resolve(sandbox, "sb-1") write_text.assert_awaited_once() written = write_text.call_args[0][0] @@ -312,22 +313,22 @@ async def test_does_not_write_file_when_no_extra_urls(self): sandbox, write_text = _mock_sandbox_with_filesystem() with patch.object( - SandboxManager, + SandboxTunnels, "_resolve_tunnels", new_callable=AsyncMock, return_value={}, ): - _cs, _vnc, _ttyd, extra = await SandboxManager._resolve_and_setup_tunnels( - sandbox, - "sb-1", - False, - False, - False, - [3000], - code_server_port=CODE_SERVER_PORT, - novnc_port=NOVNC_PORT, - ttyd_proxy_port=TTYD_PROXY_PORT, - ) + _cs, _vnc, _ttyd, extra = await SandboxTunnels( + code_server_enabled=False, + vnc_enabled=False, + settings={ + "terminalEnabled": False, + "tunnelPorts": [3000], + "codeServerPort": CODE_SERVER_PORT, + "vncPort": NOVNC_PORT, + "terminalPort": TTYD_PROXY_PORT, + }, + ).resolve(sandbox, "sb-1") assert extra is None write_text.assert_not_awaited() @@ -338,22 +339,22 @@ async def test_does_not_write_file_for_only_reserved_ports(self): sandbox, write_text = _mock_sandbox_with_filesystem() with patch.object( - SandboxManager, + SandboxTunnels, "_resolve_tunnels", new_callable=AsyncMock, return_value={CODE_SERVER_PORT: "https://cs.example.com"}, ): - await SandboxManager._resolve_and_setup_tunnels( - sandbox, - "sb-1", - True, - False, - False, - [], - code_server_port=CODE_SERVER_PORT, - novnc_port=NOVNC_PORT, - ttyd_proxy_port=TTYD_PROXY_PORT, - ) + await SandboxTunnels( + code_server_enabled=True, + vnc_enabled=False, + settings={ + "terminalEnabled": False, + "tunnelPorts": [], + "codeServerPort": CODE_SERVER_PORT, + "vncPort": NOVNC_PORT, + "terminalPort": TTYD_PROXY_PORT, + }, + ).resolve(sandbox, "sb-1") write_text.assert_not_awaited() @@ -364,24 +365,24 @@ async def test_write_failure_does_not_block_return(self): with ( patch.object( - SandboxManager, + SandboxTunnels, "_resolve_tunnels", new_callable=AsyncMock, return_value={3000: "https://tunnel-3000.example.com"}, ), - patch("src.sandbox.manager.log"), + patch("src.sandbox.tunnels.log"), ): - _cs, _vnc, _ttyd, extra = await SandboxManager._resolve_and_setup_tunnels( - sandbox, - "sb-1", - False, - False, - False, - [3000], - code_server_port=CODE_SERVER_PORT, - novnc_port=NOVNC_PORT, - ttyd_proxy_port=TTYD_PROXY_PORT, - ) + _cs, _vnc, _ttyd, extra = await SandboxTunnels( + code_server_enabled=False, + vnc_enabled=False, + settings={ + "terminalEnabled": False, + "tunnelPorts": [3000], + "codeServerPort": CODE_SERVER_PORT, + "vncPort": NOVNC_PORT, + "terminalPort": TTYD_PROXY_PORT, + }, + ).resolve(sandbox, "sb-1") assert extra == {3000: "https://tunnel-3000.example.com"} @@ -403,11 +404,11 @@ class FakeSandbox: return FakeSandbox() fake_create_aio.aio = fake_create_aio - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", fake_create_aio) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", fake_create_aio) monkeypatch.setattr( - SandboxManager, - "_resolve_and_setup_tunnels", - AsyncMock(return_value=(None, None, None, None)), + SandboxTunnels, + "resolve", + AsyncMock(return_value=TunnelUrls(None, None, None, None)), ) manager = SandboxManager() @@ -435,11 +436,11 @@ class FakeSandbox: return FakeSandbox() fake_create_aio.aio = fake_create_aio - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", fake_create_aio) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", fake_create_aio) monkeypatch.setattr( - SandboxManager, - "_resolve_and_setup_tunnels", - AsyncMock(return_value=(None, None, None, None)), + SandboxTunnels, + "resolve", + AsyncMock(return_value=TunnelUrls(None, None, None, None)), ) manager = SandboxManager() @@ -467,13 +468,13 @@ class FakeSandbox: fake_create_aio.aio = fake_create_aio monkeypatch.setattr( - "src.sandbox.manager.modal.Image.from_id", lambda *_a, **_kw: FakeImage() + "src.sandbox.launch.modal.Image.from_id", lambda *_a, **_kw: FakeImage() ) - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", fake_create_aio) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", fake_create_aio) monkeypatch.setattr( - SandboxManager, - "_resolve_and_setup_tunnels", - AsyncMock(return_value=(None, None, None, None)), + SandboxTunnels, + "resolve", + AsyncMock(return_value=TunnelUrls(None, None, None, None)), ) manager = SandboxManager() @@ -486,128 +487,61 @@ class FakeSandbox: assert captured["env"][EXPECTED_TUNNEL_PORTS_ENV_VAR] == "3000" -class TestCollectExposedPorts: - """SandboxManager._collect_exposed_ports tests.""" - - def test_no_ports_when_no_settings(self): - exposed, tunnel = SandboxManager._collect_exposed_ports( - False, False, False, None, CODE_SERVER_PORT, NOVNC_PORT, TTYD_PROXY_PORT - ) - assert exposed == [] - assert tunnel == [] - - def test_code_server_only(self): - exposed, tunnel = SandboxManager._collect_exposed_ports( - True, False, False, None, CODE_SERVER_PORT, NOVNC_PORT, TTYD_PROXY_PORT - ) - assert exposed == [CODE_SERVER_PORT] - assert tunnel == [] - - def test_tunnel_ports_only(self): - exposed, tunnel = SandboxManager._collect_exposed_ports( +@pytest.mark.parametrize( + "code_server, settings, exposed, expected_extras", + [ + (False, None, [], None), + (True, None, [CODE_SERVER_PORT], None), + (False, {"tunnelPorts": [3000, 5173]}, [3000, 5173], "3000,5173"), + (True, {"tunnelPorts": [3000]}, [CODE_SERVER_PORT, 3000], "3000"), + (False, {"terminalEnabled": True}, [TTYD_PROXY_PORT], None), + ( False, - False, - False, - {"tunnelPorts": [3000, 5173]}, - CODE_SERVER_PORT, - NOVNC_PORT, - TTYD_PROXY_PORT, - ) - assert exposed == [3000, 5173] - assert tunnel == [3000, 5173] - - def test_combined_code_server_and_tunnels(self): - exposed, tunnel = SandboxManager._collect_exposed_ports( + {"terminalEnabled": True, "tunnelPorts": [TTYD_PROXY_PORT, 3000]}, + [TTYD_PROXY_PORT, 3000], + "3000", + ), + (True, {"tunnelPorts": [CODE_SERVER_PORT, 3000]}, [CODE_SERVER_PORT, 3000], "3000"), + ( True, + {"codeServerPort": 8081, "tunnelPorts": [CODE_SERVER_PORT]}, + [8081, CODE_SERVER_PORT], + str(CODE_SERVER_PORT), + ), + ( False, - False, - {"tunnelPorts": [3000]}, - CODE_SERVER_PORT, - NOVNC_PORT, - TTYD_PROXY_PORT, - ) - assert exposed == [CODE_SERVER_PORT, 3000] - assert tunnel == [3000] - - def test_terminal_only(self): - exposed, tunnel = SandboxManager._collect_exposed_ports( - False, False, True, None, CODE_SERVER_PORT, NOVNC_PORT, TTYD_PROXY_PORT - ) - assert exposed == [TTYD_PROXY_PORT] - assert tunnel == [] - - def test_deduplicates_ttyd_port_from_tunnels(self): - exposed, tunnel = SandboxManager._collect_exposed_ports( - False, - False, - True, - {"tunnelPorts": [TTYD_PROXY_PORT, 3000]}, - CODE_SERVER_PORT, - NOVNC_PORT, - TTYD_PROXY_PORT, - ) - assert exposed == [TTYD_PROXY_PORT, 3000] - assert tunnel == [3000] - - def test_deduplicates_code_server_port_from_tunnels(self): - exposed, tunnel = SandboxManager._collect_exposed_ports( - True, - False, - False, - {"tunnelPorts": [CODE_SERVER_PORT, 3000]}, - CODE_SERVER_PORT, - NOVNC_PORT, - TTYD_PROXY_PORT, - ) - assert exposed == [CODE_SERVER_PORT, 3000] - assert tunnel == [3000] - - def test_custom_code_server_port_frees_default_for_tunnel(self): - # code-server moved to 8081 → the default 8080 is free as a user tunnel. - exposed, tunnel = SandboxManager._collect_exposed_ports( - True, - False, - False, - {"tunnelPorts": [CODE_SERVER_PORT]}, - 8081, - NOVNC_PORT, - TTYD_PROXY_PORT, - ) - assert exposed == [8081, CODE_SERVER_PORT] - assert tunnel == [CODE_SERVER_PORT] - - def test_custom_terminal_port_frees_default_for_tunnel(self): - exposed, tunnel = SandboxManager._collect_exposed_ports( - False, - False, - True, - {"tunnelPorts": [TTYD_PROXY_PORT, 3000]}, - CODE_SERVER_PORT, - NOVNC_PORT, - 7000, - ) - assert exposed == [7000, TTYD_PROXY_PORT, 3000] - assert tunnel == [TTYD_PROXY_PORT, 3000] + {"terminalEnabled": True, "terminalPort": 7000, "tunnelPorts": [TTYD_PROXY_PORT, 3000]}, + [7000, TTYD_PROXY_PORT, 3000], + f"{TTYD_PROXY_PORT},3000", + ), + ], +) +def test_exposed_ports_and_runtime_expectations_agree( + code_server, settings, exposed, expected_extras +): + tunnels = SandboxTunnels(code_server_enabled=code_server, settings=settings) + assert tunnels.exposed_ports == exposed + assert tunnels.environment.get(EXPECTED_TUNNEL_PORTS_ENV_VAR) == expected_extras class TestValidatePorts: - """SandboxManager._validate_ports tests.""" + """SandboxTunnels._validate_ports tests.""" def test_accepts_valid_ports(self): - assert SandboxManager._validate_ports([80, 3000, 65535]) == [80, 3000, 65535] + assert SandboxTunnels._validate_ports([80, 3000, 65535]) == [80, 3000, 65535] def test_rejects_out_of_range(self): - assert SandboxManager._validate_ports([0, -1, 65536, 3000]) == [3000] + assert SandboxTunnels._validate_ports([0, -1, 65536, 3000]) == [3000] def test_rejects_non_integers(self): - assert SandboxManager._validate_ports(["3000", 3.5, None, 8080]) == [8080] + assert SandboxTunnels._validate_ports(["3000", 3.5, None, 8080]) == [8080] def test_caps_at_ten(self): ports = list(range(1, 20)) - assert len(SandboxManager._validate_ports(ports)) == 10 + assert len(SandboxTunnels._validate_ports(ports)) == 10 def test_empty_list(self): - assert SandboxManager._validate_ports([]) == [] + assert SandboxTunnels._validate_ports([]) == [] def _patch_sandbox_create(monkeypatch, captured: dict) -> None: @@ -623,40 +557,40 @@ class FakeSandbox: return FakeSandbox() fake_create_aio.aio = fake_create_aio - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", fake_create_aio) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", fake_create_aio) monkeypatch.setattr( - SandboxManager, - "_resolve_and_setup_tunnels", - AsyncMock(return_value=(None, None, None, None)), + SandboxTunnels, + "resolve", + AsyncMock(return_value=TunnelUrls(None, None, None, None)), ) class TestResolveServicePorts: - """SandboxManager._resolve_service_ports tests.""" + """SandboxTunnels._resolve_service_ports tests.""" def test_defaults_when_unset(self): - assert SandboxManager._resolve_service_ports(None) == ( + assert SandboxTunnels._resolve_service_ports(None) == ( CODE_SERVER_PORT, NOVNC_PORT, TTYD_PROXY_PORT, ) - assert SandboxManager._resolve_service_ports({}) == ( + assert SandboxTunnels._resolve_service_ports({}) == ( CODE_SERVER_PORT, NOVNC_PORT, TTYD_PROXY_PORT, ) def test_uses_configured_ports(self): - assert SandboxManager._resolve_service_ports( + assert SandboxTunnels._resolve_service_ports( {"codeServerPort": 9000, "vncPort": 9001, "terminalPort": 9002} ) == (9000, 9001, 9002) def test_falls_back_on_invalid(self): - assert SandboxManager._resolve_service_ports( + assert SandboxTunnels._resolve_service_ports( {"codeServerPort": 0, "vncPort": -1, "terminalPort": 99999} ) == (CODE_SERVER_PORT, NOVNC_PORT, TTYD_PROXY_PORT) # strings and bools are not valid in-range ints - assert SandboxManager._resolve_service_ports( + assert SandboxTunnels._resolve_service_ports( {"codeServerPort": "8081", "vncPort": False, "terminalPort": True} ) == (CODE_SERVER_PORT, NOVNC_PORT, TTYD_PROXY_PORT) diff --git a/packages/modal-infra/tests/test_vnc.py b/packages/modal-infra/tests/test_vnc.py index cca7044ac6..18e2412789 100644 --- a/packages/modal-infra/tests/test_vnc.py +++ b/packages/modal-infra/tests/test_vnc.py @@ -5,6 +5,7 @@ import pytest from sandbox_runtime.constants import ( + EXPECTED_TUNNEL_PORTS_ENV_VAR, NOVNC_PORT, NOVNC_PORT_ENV_VAR, VNC_PASSWORD_ENV_VAR, @@ -12,6 +13,7 @@ VNC_PORT, ) from src.sandbox.manager import CODE_SERVER_PORT, TTYD_PROXY_PORT, SandboxConfig, SandboxManager +from src.sandbox.tunnels import SandboxTunnels, TunnelUrls def _patch_sandbox_create(monkeypatch, captured: dict) -> None: @@ -27,7 +29,7 @@ class FakeSandbox: fake_create = MagicMock() fake_create.aio = fake_create_aio - monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", fake_create) + monkeypatch.setattr("src.sandbox.launch.modal.Sandbox.create", fake_create) class TestCreateSandboxVnc: @@ -36,9 +38,9 @@ async def test_returns_url_and_password_and_exposes_only_novnc(self, monkeypatch captured = {} _patch_sandbox_create(monkeypatch, captured) monkeypatch.setattr( - SandboxManager, - "_resolve_and_setup_tunnels", - AsyncMock(return_value=(None, "https://vnc.example.com", None, None)), + SandboxTunnels, + "resolve", + AsyncMock(return_value=TunnelUrls(None, "https://vnc.example.com", None, None)), ) handle = await SandboxManager().create_sandbox( @@ -63,9 +65,9 @@ async def test_disabled_vnc_has_no_credentials_or_port(self, monkeypatch): captured = {} _patch_sandbox_create(monkeypatch, captured) monkeypatch.setattr( - SandboxManager, - "_resolve_and_setup_tunnels", - AsyncMock(return_value=(None, None, None, None)), + SandboxTunnels, + "resolve", + AsyncMock(return_value=TunnelUrls(None, None, None, None)), ) handle = await SandboxManager().create_sandbox( @@ -84,11 +86,13 @@ class TestRestoreSandboxVnc: async def test_generates_credentials_and_returns_them_with_url(self, monkeypatch): captured = {} _patch_sandbox_create(monkeypatch, captured) - monkeypatch.setattr("src.sandbox.manager.modal.Image.from_id", lambda *_args: MagicMock()) + monkeypatch.setattr("src.sandbox.launch.modal.Image.from_id", lambda *_args: MagicMock()) monkeypatch.setattr( - SandboxManager, - "_resolve_and_setup_tunnels", - AsyncMock(return_value=(None, "https://restored-vnc.example.com", None, None)), + SandboxTunnels, + "resolve", + AsyncMock( + return_value=TunnelUrls(None, "https://restored-vnc.example.com", None, None) + ), ) handle = await SandboxManager().restore_from_snapshot( @@ -108,37 +112,28 @@ async def test_generates_credentials_and_returns_them_with_url(self, monkeypatch async def test_resolves_custom_novnc_tunnel(): sandbox = MagicMock() with patch.object( - SandboxManager, + SandboxTunnels, "_resolve_tunnels", new_callable=AsyncMock, return_value={6081: "https://vnc.example.com"}, ) as resolve_tunnels: - result = await SandboxManager._resolve_and_setup_tunnels( - sandbox, - "sandbox-vnc", - False, - True, - False, - [], - code_server_port=CODE_SERVER_PORT, - novnc_port=6081, - ttyd_proxy_port=TTYD_PROXY_PORT, - ) + result = await SandboxTunnels( + code_server_enabled=False, + vnc_enabled=True, + settings={ + "terminalEnabled": False, + "tunnelPorts": [], + "codeServerPort": CODE_SERVER_PORT, + "vncPort": 6081, + "terminalPort": TTYD_PROXY_PORT, + }, + ).resolve(sandbox, "sandbox-vnc") resolve_tunnels.assert_awaited_once_with(sandbox, "sandbox-vnc", [6081]) assert result == (None, "https://vnc.example.com", None, None) def test_raw_vnc_port_is_never_exposed_as_an_extra_tunnel(): - exposed, extras = SandboxManager._collect_exposed_ports( - False, - False, - False, - {"tunnelPorts": [VNC_PORT, 3000]}, - CODE_SERVER_PORT, - NOVNC_PORT, - TTYD_PROXY_PORT, - ) - - assert exposed == [3000] - assert extras == [3000] + tunnels = SandboxTunnels(settings={"tunnelPorts": [VNC_PORT, 3000]}) + assert tunnels.exposed_ports == [3000] + assert tunnels.environment[EXPECTED_TUNNEL_PORTS_ENV_VAR] == "3000" From c24f04f26a79ce088eaca0e553772dc6bb665cf6 Mon Sep 17 00:00:00 2001 From: Cole Murray Date: Mon, 28 Sep 2026 19:51:54 -0700 Subject: [PATCH 03/13] fix: recover modal-vm handles after lost startup responses (#2115) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - Add an authenticated, lookup-only Modal VM resolution endpoint with typed allocation outcomes (`not_visible`, `other_generation`, `window_closed`, `race_pending`). It returns the owned generation's real VM ID, enabled access credentials, and tunnels without creating or retiring a VM. - Keep modal-vm create and restore generations alive after ambiguous startup outcomes. Retry resolution with a bounded backoff, claim the real handle through the existing startup path, and fail only on a definitive outcome or proven absence. - Reconcile pending handles asynchronously on bridge attach/readiness after a Durable Object restart. Generation- and pending-handle-guarded access writes replace the sandbox and shutdown handles without resetting the recorded lifetime; terminal access remains unavailable after restart when the auth token was lost. - Document the resolve contract and add Modal, provider, lifecycle, client, and repository regression tests. ## Reserved environment variables - VM launch now reserves `CODE_SERVER_PASSWORD`, `CODE_SERVER_PORT`, `TTYD_PROXY_PORT`, and `EXPECTED_TUNNEL_PORTS`. User environment variables and repository secrets with those names no longer reach the sandbox as user-provided values; resolution reads the VM-owned values to recover access. ## Verification - Control-plane unit tests: 330 files, 5,303 tests passed before final targeted tests; final focused recovery suites: 213 tests passed. - Control-plane integration tests: 120 files, 1,432 passed, 1 skipped (`--maxWorkers=2`). - Control-plane typecheck and lint passed. - Modal infra tests: 369 passed; Ruff check and format check passed. - Prettier check and `git diff --check` passed. Refs COL-237. --- *Created with [Open-Inspect](https://open-inspect-prod.vercel.app/session/11c07cec2c20ec539a74a8e8b3fbaff7)* ## Summary by CodeRabbit * **New Features** * VM sandboxes can be looked up when a launch response is unavailable, returning connection details so startup can continue. * During bridge reconnection, access details for a running VM can be restored without changing its existing lifetime. * Resolved sandbox details can include enabled service access information and available tunnel mappings. * Added an authenticated lookup endpoint for resolving VM sandboxes by session and sandbox ID. Lookup does not create or stop allocations. * Older or incomplete launch metadata may limit lookup results to the VM’s object ID. * **Bug Fixes** * Startup retries temporary errors and delayed VM visibility while preventing results from being applied to replaced or outdated sandboxes. * Missing, mismatched, or unavailable VM allocations now return more specific errors. --------- Co-authored-by: Cole Murray <2492022+ColeMurray@users.noreply.github.com> Co-authored-by: waclaude --- .github/workflows/ci-python.yml | 9 +- docs/MODAL_DOCKER.md | 42 ++ .../control-plane/src/sandbox/client.test.ts | 81 +++ packages/control-plane/src/sandbox/client.ts | 153 ++++- .../src/sandbox/lifecycle/manager.ts | 344 ++++++++++- .../src/sandbox/lifecycle/vm-resolve.test.ts | 560 ++++++++++++++++++ .../control-plane/src/sandbox/provider.ts | 15 + .../sandbox/providers/modal-provider.test.ts | 33 ++ .../src/sandbox/providers/modal-provider.ts | 76 ++- .../control-plane/src/session/components.ts | 7 +- .../src/session/sandbox-repository.test.ts | 24 + .../src/session/sandbox-repository.ts | 11 +- .../src/session/sandbox-shutdown.ts | 19 + packages/modal-infra/src/sandbox/launch.py | 33 +- packages/modal-infra/src/sandbox/manager.py | 37 +- packages/modal-infra/src/sandbox/tunnels.py | 26 +- .../modal-infra/src/sandbox/vm_recovery.py | 157 +++++ packages/modal-infra/src/web_api.py | 56 ++ .../modal-infra/tests/test_docker_launch.py | 10 + .../modal-infra/tests/test_sandbox_launch.py | 81 ++- .../tests/test_snapshot_timeout.py | 16 +- packages/modal-infra/tests/test_vm_resolve.py | 434 ++++++++++++++ 22 files changed, 2124 insertions(+), 100 deletions(-) create mode 100644 packages/control-plane/src/sandbox/lifecycle/vm-resolve.test.ts create mode 100644 packages/modal-infra/src/sandbox/vm_recovery.py create mode 100644 packages/modal-infra/tests/test_vm_resolve.py diff --git a/.github/workflows/ci-python.yml b/.github/workflows/ci-python.yml index f935dfd170..43597d337c 100644 --- a/.github/workflows/ci-python.yml +++ b/.github/workflows/ci-python.yml @@ -220,16 +220,13 @@ jobs: python-version: "3.12" cache: "pip" - - name: Setup frozen image lock checker + - name: Setup uv uses: astral-sh/setup-uv@v6 with: version: "0.9.7" - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -e ../sandbox-runtime - pip install -e ".[dev]" + run: uv sync --frozen --extra dev - name: Run tests - run: pytest tests/ -v + run: uv run --frozen --extra dev pytest tests/ -v diff --git a/docs/MODAL_DOCKER.md b/docs/MODAL_DOCKER.md index 675f07447c..56af3182d7 100644 --- a/docs/MODAL_DOCKER.md +++ b/docs/MODAL_DOCKER.md @@ -69,6 +69,48 @@ its generation is older than the materialization bound: the launch window plus t older launch that materializes later may briefly block a replacement, but the single allocation name and fenced credentials prevent overlapping work. +The authenticated `POST /api-resolve-vm-sandbox` endpoint is lookup-only. Its body contains exactly +`{"session_id":"...","sandbox_id":"..."}`; it accepts no launch settings or secrets. It finds the +running allocation by session name, checks the generation's ownership tags, and returns: + +```json +{ + "success": true, + "data": { + "sandbox_id": "generation-id", + "modal_object_id": "sb-real-id", + "code_server_url": null, + "code_server_password": null, + "vnc_url": null, + "vnc_password": null, + "ttyd_url": null, + "tunnel_urls": null, + "sandbox_backend": "modal-vm" + } +} +``` + +New VM allocations record versioned service flags and effective ports in provider-owned launch tags. +Only services enabled by these tags return URLs/passwords; extra tunnels use port-to-URL mappings. +Legacy allocations without these tags (or with unknown/incomplete metadata) resolve only the real +`modal_object_id`, not access credentials or tunnels. Resolve never infers enabled services from +environment variables, which may have contained user secrets on older allocations. Such VMs need a +new launch to recover interactive access. Resolve neither creates nor retires an allocation or +writes tunnel configuration. A stopped VM is not discoverable by name. Resolve does not return a +terminal access token; the control plane mints that token only when it still holds the generation's +sandbox auth token in memory. + +Create, restore, and resolve report typed HTTP 409 error `detail` values: + +- `not_visible`: resolve found no named allocation. +- `other_generation`: the ownership tags do not match. +- `window_closed`: create/restore missed the launch deadline with no owned allocation. +- `race_pending`: create/restore cannot yet see the winner after `AlreadyExistsError`, or resolve + found a VM whose enabled tunnel URLs are not all visible yet. + +Unexpected provider errors remain 500. The pending-reference stop endpoint retains its separate +`pending_reference_not_visible` response. + ## Switching backends Changing `SANDBOX_PROVIDER` is an operator cutover, not session migration. Existing sessions and diff --git a/packages/control-plane/src/sandbox/client.test.ts b/packages/control-plane/src/sandbox/client.test.ts index 2f122a60f1..8ddcffc694 100644 --- a/packages/control-plane/src/sandbox/client.test.ts +++ b/packages/control-plane/src/sandbox/client.test.ts @@ -137,6 +137,87 @@ describe("ModalClient", () => { }); }); + it("resolves a VM using only generation identity and preserves typed errors", async () => { + const fetchMock = vi + .spyOn(globalThis, "fetch") + .mockResolvedValueOnce( + Response.json({ + success: true, + data: { + sandbox_id: "generation", + modal_object_id: "sb-real", + sandbox_backend: "modal-vm", + code_server_url: "https://editor.example", + code_server_password: "password", + }, + }) + ) + .mockResolvedValueOnce(Response.json({ detail: "not_visible" }, { status: 409 })); + const client = createModalClient("secret", "acme"); + expect( + await client.resolveVmSandbox({ sessionId: "session", sandboxId: "generation" }) + ).toMatchObject({ + sandboxId: "generation", + modalObjectId: "sb-real", + codeServerPassword: "password", + }); + expect(String(fetchMock.mock.calls[0][0])).toBe( + "https://acme--open-inspect-api-resolve-vm-sandbox.modal.run" + ); + expect(JSON.parse(fetchMock.mock.calls[0][1]?.body as string)).toEqual({ + session_id: "session", + sandbox_id: "generation", + }); + await expect( + client.resolveVmSandbox({ sessionId: "session", sandboxId: "generation" }) + ).rejects.toMatchObject({ + status: 409, + detail: "not_visible", + }); + }); + + it.each([ + ["server error", Response.json({ detail: "Internal server error" }, { status: 500 })], + ["invalid success", Response.json({ success: true, data: {} })], + ["truncated success", new Response("{", { status: 200 })], + ])("types a VM startup %s as an unknown outcome after dispatch", async (_case, response) => { + vi.spyOn(globalThis, "fetch").mockResolvedValue(response); + const client = createModalClient("secret", "acme"); + await expect( + client.createSandbox({ + sessionId: "session", + sandboxId: "generation", + sandboxBackend: "modal-vm", + repoOwner: null, + repoName: null, + controlPlaneUrl: "https://control.test", + sandboxAuthToken: "token", + harness: "opencode", + }) + ).rejects.toMatchObject({ name: "ModalVmStartupError", outcome: "unknown" }); + }); + + it("types a VM launch-window rejection without treating it as unknown", async () => { + vi.spyOn(globalThis, "fetch").mockResolvedValue( + Response.json({ detail: "window_closed" }, { status: 409 }) + ); + await expect( + createModalClient("secret", "acme").restoreSandbox({ + snapshotImageId: "image", + sessionId: "session", + sandboxId: "generation", + sandboxBackend: "modal-vm", + sandboxAuthToken: "token", + controlPlaneUrl: "https://control.test", + repoOwner: null, + repoName: null, + harness: "opencode", + provider: "anthropic", + model: "test", + }) + ).rejects.toMatchObject({ name: "ModalVmStartupError", outcome: "window_closed" }); + }); + it("times out image-build creation when response headers stall", async () => { vi.useFakeTimers(); let markFetchStarted!: () => void; diff --git a/packages/control-plane/src/sandbox/client.ts b/packages/control-plane/src/sandbox/client.ts index ab155d4327..e552187299 100644 --- a/packages/control-plane/src/sandbox/client.ts +++ b/packages/control-plane/src/sandbox/client.ts @@ -65,6 +65,12 @@ const restoreSandboxModalResponseSchema = z.object({ }), }); +const resolveVmSandboxModalResponseSchema = restoreSandboxModalResponseSchema.extend({ + data: restoreSandboxModalResponseSchema.shape.data.extend({ + modal_object_id: z.string().min(1), + }), +}); + const snapshotSandboxModalResponseSchema = z.object({ success: z.literal(true), data: z.object({ @@ -234,6 +240,13 @@ export interface RestoreSandboxResponse { tunnelUrls?: Record; } +export interface ResolveVmSandboxRequest { + sessionId: string; + sandboxId: string; +} + +export type ResolveVmSandboxResponse = RestoreSandboxResponse & { modalObjectId: string }; + export interface SnapshotSandboxRequest { providerObjectId: string; sessionId: string; @@ -317,6 +330,23 @@ export class ModalApiError extends Error { } } +export type ModalVmStartupOutcome = + | "unknown" + | "not_visible" + | "other_generation" + | "window_closed" + | "race_pending"; + +export class ModalVmStartupError extends Error { + constructor( + public readonly outcome: ModalVmStartupOutcome, + public readonly cause: Error + ) { + super(cause.message); + this.name = "ModalVmStartupError"; + } +} + /** * Modal sandbox API client. * @@ -328,6 +358,7 @@ export class ModalClient { private snapshotVmSandboxUrl: string; private snapshotBuildSandboxUrl: string; private restoreSandboxUrl: string; + private resolveVmSandboxUrl: string; private stopSandboxUrl: string; private createImageBuildSandboxUrl: string; private startImageBuildSandboxUrl: string; @@ -342,40 +373,67 @@ export class ModalClient { schema: z.ZodType, correlation: CorrelationContext | undefined, callerSignal: AbortSignal | undefined, - onResponse: (status: number) => void + onResponse: (status: number) => void, + vmStartup = false ): Promise { const headers = await this.getPostHeaders(correlation); - return withRequestDeadline("Modal", endpoint, deadlineMs, callerSignal, async (signal) => { - const response = await fetch(url, { - method: "POST", - headers, - signal, - body: JSON.stringify(body), - }); - onResponse(response.status); - if (!response.ok) { - const text = await response.text(); - let body: unknown; - try { - body = JSON.parse(text); - } catch { - // Non-JSON provider responses still retain their status and raw text. + const payload = JSON.stringify(body); + try { + return await withRequestDeadline( + "Modal", + endpoint, + deadlineMs, + callerSignal, + async (signal) => { + const response = await fetch(url, { + method: "POST", + headers, + signal, + body: payload, + }); + onResponse(response.status); + if (!response.ok) { + const text = await response.text(); + let body: unknown; + try { + body = JSON.parse(text); + } catch { + // Non-JSON provider responses still retain their status and raw text. + } + const detail = + body !== null && + typeof body === "object" && + "detail" in body && + typeof body.detail === "string" + ? body.detail + : undefined; + throw new ModalApiError( + `Modal API error: ${response.status} ${text}`, + response.status, + detail + ); + } + return parseModalApiResponse(schema, await response.json()); } - const detail = - body !== null && - typeof body === "object" && - "detail" in body && - typeof body.detail === "string" - ? body.detail - : undefined; - throw new ModalApiError( - `Modal API error: ${response.status} ${text}`, - response.status, - detail - ); + ); + } catch (error) { + if (!vmStartup) throw error; + if (error instanceof ModalApiError) { + const detail = error.detail; + if ( + detail === "not_visible" || + detail === "other_generation" || + detail === "window_closed" || + detail === "race_pending" + ) + throw new ModalVmStartupError(detail, error); + if (error.status < 500) throw error; } - return parseModalApiResponse(schema, await response.json()); - }); + throw new ModalVmStartupError( + "unknown", + error instanceof Error ? error : new Error(String(error)) + ); + } } constructor(secret: string, workspace: string, environmentWebSuffix?: string, apiUrl?: string) { @@ -393,6 +451,7 @@ export class ModalClient { this.snapshotVmSandboxUrl = url("api-snapshot-vm-sandbox"); this.snapshotBuildSandboxUrl = url("api-snapshot-build-sandbox"); this.restoreSandboxUrl = url("api-restore-sandbox"); + this.resolveVmSandboxUrl = url("api-resolve-vm-sandbox"); this.stopSandboxUrl = url("api-stop-sandbox"); this.createImageBuildSandboxUrl = url("api-create-build-sandbox"); this.startImageBuildSandboxUrl = url("api-start-build-sandbox"); @@ -467,7 +526,8 @@ export class ModalClient { createSandboxModalResponseSchema, correlation, request.signal, - (status) => (httpStatus = status) + (status) => (httpStatus = status), + request.sandboxBackend === "modal-vm" ); outcome = "success"; @@ -534,7 +594,8 @@ export class ModalClient { restoreSandboxModalResponseSchema, correlation, request.signal, - (status) => (httpStatus = status) + (status) => (httpStatus = status), + request.sandboxBackend === "modal-vm" ); outcome = "success"; @@ -564,6 +625,34 @@ export class ModalClient { } } + /** Lookup-only recovery of a generation's named Modal VM allocation. */ + async resolveVmSandbox( + request: ResolveVmSandboxRequest, + correlation?: CorrelationContext + ): Promise { + const result = await this.postJson( + this.resolveVmSandboxUrl, + "resolveVmSandbox", + MODAL_SANDBOX_START_REQUEST_DEADLINE_MS, + { session_id: request.sessionId, sandbox_id: request.sandboxId }, + resolveVmSandboxModalResponseSchema, + correlation, + undefined, + () => {} + ); + return { + sandboxId: result.data.sandbox_id, + modalObjectId: result.data.modal_object_id, + sandboxBackend: result.data.sandbox_backend, + codeServerUrl: result.data.code_server_url ?? undefined, + codeServerPassword: result.data.code_server_password ?? undefined, + vncUrl: result.data.vnc_url ?? undefined, + vncPassword: result.data.vnc_password ?? undefined, + ttydUrl: result.data.ttyd_url ?? undefined, + tunnelUrls: result.data.tunnel_urls ?? undefined, + }; + } + /** Trigger a filesystem snapshot for a sandbox object. */ async snapshotSandbox( request: SnapshotSandboxRequest, diff --git a/packages/control-plane/src/sandbox/lifecycle/manager.ts b/packages/control-plane/src/sandbox/lifecycle/manager.ts index 81a85f8b76..4156c6bd65 100644 --- a/packages/control-plane/src/sandbox/lifecycle/manager.ts +++ b/packages/control-plane/src/sandbox/lifecycle/manager.ts @@ -55,6 +55,7 @@ import { DEFAULT_HEARTBEAT_CONFIG, DEFAULT_CONNECTING_TIMEOUT_CONFIG, DEFAULT_BOOT_BUDGET_CONFIG, + PENDING_VM_REFERENCE_MATERIALIZE_BOUND_MS, type CircuitBreakerConfig, type SpawnConfig, } from "./decisions"; @@ -71,7 +72,7 @@ import { type ImageBuildLookup, type SelectedImageBuild, } from "./image-selection"; -import type { AlarmScheduler, SessionWebSocket } from "../../platform-ports"; +import type { AlarmScheduler, BackgroundTasks, SessionWebSocket } from "../../platform-ports"; import { DEFAULT_SANDBOX_STATUS } from "../sandbox-status"; import type { SandboxGeneration, @@ -86,6 +87,9 @@ import type { SandboxPushAdmission, } from "./ports"; import { shutdownPolicyForLaunch, type ShutdownLifecyclePolicy } from "./shutdown-policy"; +import { parsePendingVmReference } from "../providers/pending-vm-reference"; +import { ModalApiError, ModalVmStartupError } from "../client"; +import type { ResolveSandboxResult } from "../provider"; export type { SandboxGeneration, SandboxAlarmResult } from "./ports"; export type { ImageBuildLookup } from "./image-selection"; @@ -97,6 +101,14 @@ const log = createLogger("lifecycle-manager"); const TERMINAL_TOKEN_TTL_SECONDS = 86400; const PROVIDER_REPLACEMENT_STOP_TIMEOUT_MS = 10_000; const REJECTED_ALLOCATION_CLEANUP_RETRY_MS = 30_000; +const VM_RESOLVE_RETRY_MS = 10_000; + +function vmAllocationDetail(error: unknown): string | undefined { + const cause = error instanceof SandboxProviderError ? error.cause : error; + if (cause instanceof ModalVmStartupError) return cause.outcome; + if (cause instanceof ModalApiError) return cause.detail; + return undefined; +} // ==================== Dependency Interfaces ==================== @@ -118,6 +130,12 @@ export interface SandboxShutdownLifecycle { ): Promise<"registered" | "expired" | "superseded">; /** Records the provider-confirmed handle and scheduling lifetime after startup. */ recordProviderStartup(generation: SandboxGeneration, lifetime: SandboxLifetime): Promise; + /** Swap a pending handle after bridge recovery without changing shutdown policy or lifetime. */ + recordResolvedProviderHandle?( + generation: SandboxGeneration, + expectedReference: string, + providerObjectId: string + ): void; /** Blocks generic destructive lifecycle work while shutdown or capture ownership is unresolved. */ isHolding(): boolean; /** Tells a runtime refused at reconnect to retry while a capture needs its sandbox. */ @@ -270,7 +288,8 @@ export interface SandboxStorage { vnc: { url: string; password: string } | null; ttyd: { url: string | null; token: string } | null; tunnelUrls: Record | null; - } + }, + expectedProviderObjectId?: string ): Promise; /** Update sandbox Modal object ID (for snapshot API) */ updateSandboxModalObjectId(modalObjectId: string | null): void; @@ -515,6 +534,18 @@ export class SandboxLifecycleManager private isSpawningSandbox = false; private isTerminatingSandbox = false; private providerStartupPending = false; + private bridgeResolution: SandboxGeneration | null = null; + private bridgeRetryGeneration: SandboxGeneration | null = null; + private bridgeStartupClaim: SandboxGeneration | null = null; + private bridgeResolvedStartup: { + generation: SandboxGeneration; + result: ResolveSandboxResult; + } | null = null; + private vmStartupAuth: { + generation: SandboxGeneration; + sessionId: string; + token: string; + } | null = null; retireShutdownAccess(): void { this.clearSandboxAccessState(); this.wsManager.detachSandboxWebSocket(1000, "Sandbox state preserved"); @@ -551,7 +582,8 @@ export class SandboxLifecycleManager private readonly idGenerator: IdGenerator, private readonly shutdown: SandboxShutdownLifecycle, private readonly config: SandboxLifecycleConfig, - private readonly imageBuildLookup?: ImageBuildLookup + private readonly imageBuildLookup?: ImageBuildLookup, + private readonly backgroundTasks?: BackgroundTasks ) {} /** @@ -850,10 +882,15 @@ export class SandboxLifecycleManager ...multiRepoFields, }; + if (this.provider.name === "modal-vm") + this.vmStartupAuth = { generation, sessionId, token: sandboxAuthToken }; + let result: CreateSandboxResult; try { await this.recordPendingProviderReference(generation, createConfig); - result = await this.provider.createSandbox(createConfig); + const created = await this.createWithVmRecovery(createConfig, generation); + if (!created) return; + result = created; } catch (error) { if (!selectedImage) throw error; if (!(error instanceof PrebuiltImageUnavailableError)) { @@ -884,23 +921,25 @@ export class SandboxLifecycleManager const retryNow = Math.max(Date.now(), now + 1); const retry = this.spawnGeneration(session, retryNow); generation = retry; + this.vmStartupAuth = null; ({ sandboxAuthToken, expectedSandboxId } = await this.reserveSpawnIdentity(retry, { preserveProviderObjectId: false, shutdownPolicy: shutdownPolicyForLaunch("new", null), })); - await this.recordPendingProviderReference(generation, { - ...createConfig, - sandboxId: expectedSandboxId, - generationCreatedAtMs: retry.createdAt, - }); - result = await this.provider.createSandbox({ + if (this.provider.name === "modal-vm") + this.vmStartupAuth = { generation, sessionId, token: sandboxAuthToken }; + const retryConfig: CreateSandboxConfig = { ...createConfig, sandboxId: expectedSandboxId, generationCreatedAtMs: retry.createdAt, sandboxAuthToken, prebuiltImageId: null, prebuiltImageSha: null, - }); + }; + await this.recordPendingProviderReference(generation, retryConfig); + const created = await this.createWithVmRecovery(retryConfig, generation); + if (!created) return; + result = created; } if (!(await this.claimProviderStartup(generation, result.providerObjectId, result.lifetime))) @@ -975,6 +1014,8 @@ export class SandboxLifecycleManager } finally { this.isSpawningSandbox = false; this.providerStartupPending = false; + if (this.vmStartupAuth?.generation === generation && this.bridgeStartupClaim !== generation) + this.vmStartupAuth = null; } } @@ -1264,9 +1305,23 @@ export class SandboxLifecycleManager sandboxSettings, ...multiRepoSpawnFields(repositories), }; + if (this.provider.name === "modal-vm") + this.vmStartupAuth = { + generation, + sessionId: restoreConfig.sessionId, + token: sandboxAuthToken, + }; await this.recordPendingProviderReference(generation, restoreConfig); this.shutdown.markRecoveryInvoked(generation); - const result = await this.provider.restoreFromSnapshot(restoreConfig); + let result; + try { + result = await this.provider.restoreFromSnapshot(restoreConfig); + } catch (error) { + if (!this.provider.isUnknownStartupError?.(error)) throw error; + const recovered = await this.resolveUnknownVmStartup(generation, restoreConfig); + if (!recovered) return; + result = { ...recovered, success: true as const }; + } if (result.success) { if ( @@ -1355,6 +1410,8 @@ export class SandboxLifecycleManager } finally { this.isSpawningSandbox = false; this.providerStartupPending = false; + if (this.vmStartupAuth?.generation === generation && this.bridgeStartupClaim !== generation) + this.vmStartupAuth = null; } } @@ -2164,6 +2221,7 @@ export class SandboxLifecycleManager if (!row) return false; const generation = { sandboxId: row.modal_sandbox_id, createdAt: row.created_at }; if (!this.storage.markSandboxReady(generation)) return false; + this.resolvePendingBridge(generation); this.log.info("sandbox.ready", { event: "sandbox.ready", harness: harness ?? null }); this.updateLastActivity(timestamp); this.broadcaster.broadcast({ type: "sandbox_status", status: "ready" }); @@ -2380,7 +2438,18 @@ export class SandboxLifecycleManager sessionId: string, sandboxId: string ): Promise { - const token = await mintJwt( + const token = await this.mintTtydToken(sandboxAuthToken, sessionId, sandboxId); + + this.log.info("Storing ttyd info", { url }); + await this.storage.updateSandboxAccess("ttyd", url, token); + } + + private mintTtydToken( + sandboxAuthToken: string, + sessionId: string, + sandboxId: string + ): Promise { + return mintJwt( { sub: sessionId, sid: sandboxId, @@ -2389,9 +2458,6 @@ export class SandboxLifecycleManager }, sandboxAuthToken ); - - this.log.info("Storing ttyd info", { url }); - await this.storage.updateSandboxAccess("ttyd", url, token); } private async recordPendingProviderReference( @@ -2435,6 +2501,251 @@ export class SandboxLifecycleManager } } + private async createWithVmRecovery( + config: CreateSandboxConfig, + generation: SandboxGeneration + ): Promise { + try { + return await this.provider.createSandbox(config); + } catch (error) { + if (!this.provider.isUnknownStartupError?.(error)) throw error; + const recovered = await this.resolveUnknownVmStartup(generation, config); + return recovered ? { ...recovered, createdAt: generation.createdAt } : null; + } + } + + private knownBridgeStartup( + generation: SandboxGeneration, + row: SandboxRow | null + ): ResolveSandboxResult | null { + const known = this.bridgeResolvedStartup; + if ( + !known || + row?.modal_sandbox_id !== generation.sandboxId || + row.created_at !== generation.createdAt || + row.fenced || + !["spawning", "connecting", "ready"].includes(row.status) || + row.modal_object_id !== known.result.providerObjectId || + known.generation.sandboxId !== generation.sandboxId || + known.generation.createdAt !== generation.createdAt + ) + return null; + return known.result; + } + + private async resolveUnknownVmStartup( + generation: SandboxGeneration, + config: Pick< + CreateSandboxConfig, + "sessionId" | "sandboxId" | "generationCreatedAtMs" | "timeoutSeconds" + > + ): Promise { + if (!this.provider.resolveSandbox) return null; + const reference = this.provider.pendingSandboxAllocation?.(config)?.reference; + while (true) { + const row = this.storage.getSandbox(); + const bridged = this.knownBridgeStartup(generation, row); + if (bridged) return bridged; + const resolvedByBridge = + !!row?.modal_object_id && + row.modal_object_id !== reference && + parsePendingVmReference(row.modal_object_id) === null; + if ( + row?.modal_sandbox_id !== generation.sandboxId || + row.created_at !== generation.createdAt || + row.fenced || + !["spawning", "connecting", "ready"].includes(row.status) || + (row.modal_object_id !== reference && !resolvedByBridge) + ) + return null; + try { + return await this.provider.resolveSandbox({ + ...config, + generationCreatedAtMs: generation.createdAt, + }); + } catch (error) { + const detail = vmAllocationDetail(error); + if (detail === "other_generation") throw error; + if (detail !== "not_visible" && !this.provider.isUnknownStartupError?.(error)) throw error; + if (Date.now() - generation.createdAt >= PENDING_VM_REFERENCE_MATERIALIZE_BOUND_MS) { + const current = this.storage.getSandbox(); + const bridgedAfterLookup = this.knownBridgeStartup(generation, current); + if (bridgedAfterLookup) return bridgedAfterLookup; + if ( + current?.modal_sandbox_id === generation.sandboxId && + current.created_at === generation.createdAt && + !current.fenced && + current.modal_object_id && + parsePendingVmReference(current.modal_object_id) === null + ) { + const lifetime = this.provider.pendingSandboxAllocation?.(config)?.lifetime; + if (lifetime) + return { + sandboxId: config.sandboxId, + providerObjectId: current.modal_object_id, + lifetime, + }; + } + if (detail === "not_visible") + throw new SandboxProviderError( + "The VM allocation did not appear for this attempt. Please retry.", + "transient", + error instanceof Error ? error : undefined + ); + if ( + current?.modal_sandbox_id === generation.sandboxId && + current.created_at === generation.createdAt && + !current.fenced && + current.modal_object_id === reference + ) + this.bridgeStartupClaim = generation; + return null; + } + } + await new Promise((resolve) => setTimeout(resolve, VM_RESOLVE_RETRY_MS)); + } + } + + /** Bridge-triggered reconciliation is lookup-only and never holds readiness. */ + private resolvePendingBridge(generation: SandboxGeneration): void { + if (this.bridgeResolution) { + if ( + this.bridgeResolution.sandboxId !== generation.sandboxId || + this.bridgeResolution.createdAt !== generation.createdAt + ) + this.bridgeRetryGeneration = generation; + return; + } + if (!this.provider.resolveSandbox) return; + const row = this.storage.getSandbox(); + const reference = row?.modal_object_id; + const pending = reference ? parsePendingVmReference(reference) : null; + const session = this.sessionContext.getSession(); + if ( + !row || + row.fenced || + !["spawning", "connecting", "ready"].includes(row.status) || + row.created_at !== generation.createdAt || + row.modal_sandbox_id !== generation.sandboxId || + !reference || + !pending || + !session || + pending.sandboxId !== row.modal_sandbox_id || + pending.sessionId !== (session.session_name || session.id) + ) + return; + const config = { + sessionId: pending.sessionId, + sandboxId: pending.sandboxId, + generationCreatedAtMs: generation.createdAt, + timeoutSeconds: this.resolveSandboxTimeoutSeconds(this.parseSandboxSettings(session)), + }; + const retryDeadlineAtMs = Date.now() + PENDING_VM_REFERENCE_MATERIALIZE_BOUND_MS; + this.bridgeResolution = generation; + const work = () => + (async () => { + let result: ResolveSandboxResult; + while (true) { + const current = this.storage.getSandbox(); + if ( + current?.modal_sandbox_id !== generation.sandboxId || + current.created_at !== generation.createdAt || + current.fenced || + !["spawning", "connecting", "ready"].includes(current.status) || + current.modal_object_id !== reference + ) + return; + try { + result = await this.provider.resolveSandbox!(config); + break; + } catch (error) { + const detail = vmAllocationDetail(error); + if (detail !== "not_visible" && !this.provider.isUnknownStartupError?.(error)) + throw error; + if ( + Date.now() >= retryDeadlineAtMs || + (detail === "not_visible" && + Date.now() - generation.createdAt >= PENDING_VM_REFERENCE_MATERIALIZE_BOUND_MS) || + this.bridgeRetryGeneration + ) + return; + await new Promise((resolve) => setTimeout(resolve, VM_RESOLVE_RETRY_MS)); + } + } + if (!result.providerObjectId) return; + const auth = this.vmStartupAuth; + const terminalToken = + result.ttydUrl && + auth && + auth.generation.sandboxId === generation.sandboxId && + auth.generation.createdAt === generation.createdAt + ? await this.mintTtydToken(auth.token, auth.sessionId, generation.sandboxId!) + : null; + const committed = await this.storage.completeProviderResume( + generation, + { + providerObjectId: result.providerObjectId, + codeServer: + result.codeServerUrl && result.codeServerPassword + ? { url: result.codeServerUrl, password: result.codeServerPassword } + : null, + vnc: result.vncAccess ?? null, + ttyd: + result.ttydUrl && terminalToken + ? { url: result.ttydUrl, token: terminalToken } + : null, + tunnelUrls: result.tunnelUrls ?? null, + }, + reference + ); + if (!committed) return; + this.bridgeResolvedStartup = { + generation, + result: { + sandboxId: result.sandboxId, + providerObjectId: result.providerObjectId, + lifetime: result.lifetime, + }, + }; + if ( + this.bridgeStartupClaim?.sandboxId === generation.sandboxId && + this.bridgeStartupClaim.createdAt === generation.createdAt + ) { + this.bridgeStartupClaim = null; + try { + await this.claimProviderStartup(generation, result.providerObjectId, result.lifetime); + } finally { + if ( + this.vmStartupAuth?.generation.sandboxId === generation.sandboxId && + this.vmStartupAuth.generation.createdAt === generation.createdAt + ) + this.vmStartupAuth = null; + } + } else { + this.shutdown.recordResolvedProviderHandle?.( + generation, + reference, + result.providerObjectId + ); + } + this.broadcastProviderAccessIfConnected(); + })() + .catch((error) => { + this.log.warn("Bridge VM resolution failed", { + event: "sandbox.vm_resolve_failed", + error: error instanceof Error ? error.message : String(error), + }); + }) + .finally(() => { + this.bridgeResolution = null; + const queued = this.bridgeRetryGeneration; + this.bridgeRetryGeneration = null; + if (queued) this.resolvePendingBridge(queued); + }); + if (this.backgroundTasks) this.backgroundTasks.submit(work, { name: "sandbox.vm_resolve" }); + else void work(); + } + private async handleRejectedStartupAllocation( error: unknown, generation: SandboxGeneration | null @@ -2605,6 +2916,7 @@ export class SandboxLifecycleManager * in flight, and for the user, who sees the boot begin. */ onSandboxSocketAttached(generation: SandboxGeneration): void { + this.resolvePendingBridge(generation); if (this.storage.transitionSandboxStatus(generation, "spawning", "connecting")) { this.broadcaster.broadcast({ type: "sandbox_status", status: "connecting" }); return; diff --git a/packages/control-plane/src/sandbox/lifecycle/vm-resolve.test.ts b/packages/control-plane/src/sandbox/lifecycle/vm-resolve.test.ts new file mode 100644 index 0000000000..b972627940 --- /dev/null +++ b/packages/control-plane/src/sandbox/lifecycle/vm-resolve.test.ts @@ -0,0 +1,560 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { ModalApiError, type ModalClient, type ResolveVmSandboxResponse } from "../client"; +import { ModalSandboxProvider } from "../providers/modal-provider"; +import { formatPendingVmReference } from "../providers/pending-vm-reference"; +import { RequestDeadlineError } from "../request-deadline"; +import { SandboxLifecycleManager } from "./manager"; +import type { ImageBuildLookup } from "./image-selection"; +import { computeRepositoriesFingerprint } from "../../image-builds/fingerprint"; +import { PENDING_VM_REFERENCE_MATERIALIZE_BOUND_MS } from "./decisions"; +import { SandboxShutdownCoordinator } from "../../session/sandbox-shutdown"; +import type { ShutdownRecord } from "../../session/sandbox-shutdown-repository"; +import { + createMockSandbox, + createMockSession, + createMockStorage, + createMockBroadcaster, + createMockWebSocketManager, + createMockAlarmScheduler, + createMockIdGenerator, + createTestConfig, +} from "./test-helpers"; +import { COMPATIBLE_RUNTIME_VERSION } from "../../image-builds/test-helpers"; + +function fixture(action: "create" | "restore" = "create", imageBuildLookup?: ImageBuildLookup) { + const sandbox = createMockSandbox({ + status: action === "create" ? "pending" : "stopped", + snapshot_image_id: action === "restore" ? "im-saved" : null, + snapshot_runtime_version: action === "restore" ? COMPATIBLE_RUNTIME_VERSION : null, + }); + const session = createMockSession({ + code_server_enabled: 1, + vnc_enabled: 1, + sandbox_settings: JSON.stringify({ sandboxTimeoutMs: 3_600_000 }), + }); + const storage = createMockStorage( + session, + sandbox, + undefined, + imageBuildLookup ? [{ repoOwner: "testowner", repoName: "testrepo", baseBranch: "main" }] : [] + ); + const broadcaster = createMockBroadcaster(); + const wsManager = createMockWebSocketManager(); + const providerResponse: ResolveVmSandboxResponse = { + sandboxId: "unused", + modalObjectId: "sb-real", + sandboxBackend: "modal-vm", + codeServerUrl: "https://editor.example", + codeServerPassword: "editor-password", + vncUrl: "https://desktop.example", + vncPassword: "desktop-password", + ttydUrl: "https://terminal.example", + tunnelUrls: { "8080": "https://port.example" }, + }; + const client = { + createSandbox: vi.fn( + async ( + _config: unknown + ): Promise<{ + sandboxId: string; + modalObjectId: string; + sandboxBackend: string; + createdAt: number; + }> => { + throw new RequestDeadlineError("Modal", "createSandbox", 60_000); + } + ), + restoreSandbox: vi.fn(async () => { + throw new ModalApiError("pending race", 409, "race_pending"); + }), + resolveVmSandbox: vi.fn( + async (req: { sandboxId: string }): Promise => ({ + ...providerResponse, + sandboxId: req.sandboxId, + }) + ), + stopSandbox: vi.fn(async () => {}), + }; + const provider = new ModalSandboxProvider(client as unknown as ModalClient, "modal-vm"); + const backgroundTasks = { submit: vi.fn((task: () => Promise) => void task()) }; + let state: ShutdownRecord | null = null; + const store = { + read: () => (state ? structuredClone(state) : null), + write: (next: ShutdownRecord) => { + state = structuredClone(next); + }, + }; + if (action === "restore") { + store.write({ + phase: "saved", + generation: { sandboxId: sandbox.modal_sandbox_id!, createdAt: sandbox.created_at }, + provider: "modal-vm", + providerObjectId: null, + sourceRetired: true, + lifetimeKind: "none", + expiresAtMs: null, + drainAtMs: null, + generationReady: true, + lifecyclePolicy: "confirmed", + receipt: { + kind: "snapshot", + artifactId: "im-saved", + provider: "modal-vm", + savedAtMs: Date.now(), + runtimeVersion: COMPATIBLE_RUNTIME_VERSION, + }, + }); + } + const deps = { + store, + provider, + sandbox: storage, + session: { getSession: () => session, transaction: (fn: () => T) => fn() }, + messages: { getProcessingMessage: () => null }, + failures: { record: vi.fn(), deliver: vi.fn() }, + messenger: broadcaster, + sockets: { getSandboxSocket: () => null }, + alarm: createMockAlarmScheduler(), + background: { submit: vi.fn() }, + onLifecycleChange: vi.fn(async () => {}), + reconcileStatusFromMessages: vi.fn(async () => {}), + retireAccess: vi.fn(), + }; + const makeManager = () => + new SandboxLifecycleManager( + provider, + storage, + storage, + broadcaster, + wsManager, + createMockAlarmScheduler(), + createMockIdGenerator(), + new SandboxShutdownCoordinator(deps as never), + createTestConfig(), + imageBuildLookup, + backgroundTasks + ); + return { sandbox, storage, broadcaster, client, store, makeManager, wsManager, backgroundTasks }; +} + +describe("modal-vm startup resolution", () => { + afterEach(() => vi.useRealTimers()); + + it.each(["create", "restore"] as const)( + "recovers an unknown %s without resetting lifetime", + async (action) => { + vi.useFakeTimers({ toFake: ["Date"] }); + vi.setSystemTime(new Date("2030-01-01T00:00:00Z")); + const f = fixture(action); + const manager = f.makeManager(); + await manager.spawnSandbox(); + expect(f.sandbox.modal_object_id).toBe("sb-real"); + expect(f.sandbox.status).toBe("connecting"); + expect(f.store.read()).toMatchObject({ + providerObjectId: "sb-real", + expiresAtMs: f.sandbox.created_at + 3_600_000, + }); + expect(f.sandbox.code_server_url).toBe("https://editor.example"); + expect(f.sandbox.vnc_password).toBe("desktop-password"); + expect(f.sandbox.tunnel_urls).toBe(JSON.stringify({ "8080": "https://port.example" })); + expect(f.sandbox.ttyd_token).toBeTruthy(); + expect(f.broadcaster.messages).not.toContainEqual( + expect.objectContaining({ type: "sandbox_error" }) + ); + expect(f.sandbox.spawn_failure_count).toBe(0); + } + ); + + it("fails once when the allocation remains invisible past the bound, allowing respawn", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2030-01-01T00:00:00Z")); + const f = fixture(); + f.client.resolveVmSandbox.mockRejectedValue(new ModalApiError("invisible", 409, "not_visible")); + const manager = f.makeManager(); + const spawning = manager.spawnSandbox(); + await vi.waitFor(() => expect(f.client.resolveVmSandbox).toHaveBeenCalledOnce()); + expect(f.sandbox.status).toBe("spawning"); + await vi.advanceTimersByTimeAsync(PENDING_VM_REFERENCE_MATERIALIZE_BOUND_MS + 16_000); + await spawning; + expect(f.sandbox.status).toBe("failed"); + expect(f.broadcaster.messages).toContainEqual( + expect.objectContaining({ type: "sandbox_error" }) + ); + expect(f.storage.transitionSandboxStatus).toHaveBeenCalledTimes(1); + f.client.createSandbox.mockResolvedValueOnce({ + sandboxId: "next", + modalObjectId: "sb-next", + sandboxBackend: "modal-vm", + createdAt: Date.now(), + }); + await manager.spawnSandbox(); + expect(f.client.createSandbox).toHaveBeenCalledTimes(2); + }); + + it("fails definitively for another generation", async () => { + const f = fixture(); + f.client.resolveVmSandbox.mockRejectedValue( + new ModalApiError("occupied", 409, "other_generation") + ); + await f.makeManager().spawnSandbox(); + expect(f.sandbox.status).toBe("failed"); + expect(f.client.resolveVmSandbox).toHaveBeenCalledOnce(); + }); + + it("resolves an ambiguous base-image retry after a prebuilt image is unavailable", async () => { + const imageBuildLookup: ImageBuildLookup = { + getLatestReady: vi.fn(async () => ({ + id: "image-build-1", + provider_image_id: "im-prebuilt", + repositories_fingerprint: await computeRepositoriesFingerprint([ + { repoOwner: "testowner", repoName: "testrepo", baseBranch: "main" }, + ]), + repository_shas: JSON.stringify([ + { repoOwner: "testowner", repoName: "testrepo", baseSha: "sha-1" }, + ]), + runtime_version: COMPATIBLE_RUNTIME_VERSION, + })), + markRestoreFailed: vi.fn(async () => true), + }; + const f = fixture("create", imageBuildLookup); + f.client.createSandbox + .mockRejectedValueOnce(new ModalApiError("prebuilt unavailable", 410)) + .mockRejectedValueOnce(new RequestDeadlineError("Modal", "createSandbox", 60_000)); + await f.makeManager().spawnSandbox(); + expect(f.client.createSandbox).toHaveBeenCalledTimes(2); + expect(f.client.createSandbox.mock.calls[1][0]).toMatchObject({ prebuiltImageId: null }); + expect(imageBuildLookup.markRestoreFailed).toHaveBeenCalledOnce(); + expect(f.client.resolveVmSandbox).toHaveBeenCalledOnce(); + expect(f.sandbox.modal_object_id).toBe("sb-real"); + }); + + it("bounds repeated transient lookup errors without failing the pending generation", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2030-01-01T00:00:00Z")); + const f = fixture(); + f.client.resolveVmSandbox.mockRejectedValue(new ModalApiError("unavailable", 503)); + const manager = f.makeManager(); + const spawning = manager.spawnSandbox(); + await vi.waitFor(() => expect(f.client.resolveVmSandbox).toHaveBeenCalledOnce()); + await vi.advanceTimersByTimeAsync(PENDING_VM_REFERENCE_MATERIALIZE_BOUND_MS + 20_000); + await spawning; + expect(manager.isProviderStartupPending()).toBe(false); + expect(f.sandbox.status).toBe("spawning"); + expect(f.sandbox.modal_object_id).toBe( + formatPendingVmReference("test-session", f.sandbox.modal_sandbox_id!) + ); + expect(f.broadcaster.messages).not.toContainEqual( + expect.objectContaining({ type: "sandbox_error" }) + ); + }); + + it("completes a restore after bounded transient errors when its bridge later resolves", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2030-01-01T00:00:00Z")); + const f = fixture("restore"); + const manager = f.makeManager(); + f.client.resolveVmSandbox.mockRejectedValue(new ModalApiError("unavailable", 503)); + const restoring = manager.spawnSandbox(); + await vi.waitFor(() => expect(f.client.resolveVmSandbox).toHaveBeenCalledOnce()); + await vi.advanceTimersByTimeAsync(PENDING_VM_REFERENCE_MATERIALIZE_BOUND_MS + 10_000); + await restoring; + expect(f.store.read()).toMatchObject({ phase: "restoring", restoreInvoked: true }); + f.client.resolveVmSandbox.mockResolvedValue({ + sandboxId: f.sandbox.modal_sandbox_id!, + modalObjectId: "sb-real", + sandboxBackend: "modal-vm", + ttydUrl: "https://terminal.example", + }); + const generation = { sandboxId: f.sandbox.modal_sandbox_id!, createdAt: f.sandbox.created_at }; + manager.onSandboxSocketAttached(generation); + await vi.waitFor(() => expect(f.store.read()?.phase).toBe("running")); + expect(manager.onRuntimeReady(Date.now(), undefined, 1)).toBe(true); + manager.onShutdownGenerationReady({ + type: "sandbox_generation_ready", + generation, + sandboxId: generation.sandboxId, + timestamp: Date.now(), + }); + expect(f.sandbox.modal_object_id).toBe("sb-real"); + expect(f.sandbox.ttyd_token).toBeTruthy(); + expect(manager.pushAdmissionDecision()).toBe("ready"); + }); + + it("claims a bridge-resolved restore without a second network lookup", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2030-01-01T00:00:00Z")); + const f = fixture("restore"); + let rejectRestore!: (error: Error) => void; + f.client.restoreSandbox.mockImplementationOnce( + () => new Promise((_resolve, reject) => (rejectRestore = reject)) + ); + const manager = f.makeManager(); + const restoring = manager.spawnSandbox(); + await vi.waitFor(() => expect(f.client.restoreSandbox).toHaveBeenCalledOnce()); + const generation = { sandboxId: f.sandbox.modal_sandbox_id!, createdAt: f.sandbox.created_at }; + manager.onSandboxSocketAttached(generation); + await vi.waitFor(() => expect(f.sandbox.modal_object_id).toBe("sb-real")); + rejectRestore(new ModalApiError("pending race", 409, "race_pending")); + await restoring; + expect(f.client.resolveVmSandbox).toHaveBeenCalledOnce(); + expect(f.store.read()).toMatchObject({ phase: "running", providerObjectId: "sb-real" }); + expect(f.store.read()?.restoreInvoked).toBeUndefined(); + expect(f.sandbox.ttyd_token).toBeTruthy(); + }); + + it("claims a bridge result committed during the final inconclusive lookup", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2030-01-01T00:00:00Z")); + const f = fixture("restore"); + let rejectLookup!: (error: Error) => void; + f.client.resolveVmSandbox.mockImplementationOnce( + () => new Promise((_resolve, reject) => (rejectLookup = reject)) + ); + const manager = f.makeManager(); + const restoring = manager.spawnSandbox(); + await vi.waitFor(() => expect(f.client.resolveVmSandbox).toHaveBeenCalledOnce()); + const generation = { sandboxId: f.sandbox.modal_sandbox_id!, createdAt: f.sandbox.created_at }; + f.sandbox.status = "connecting"; + const pending = f.sandbox.modal_object_id!; + f.sandbox.modal_object_id = "sb-real"; + f.store.write({ ...f.store.read()!, providerObjectId: "sb-real" }); + vi.setSystemTime(generation.createdAt + PENDING_VM_REFERENCE_MATERIALIZE_BOUND_MS); + rejectLookup(new ModalApiError("invisible", 409, "not_visible")); + await restoring; + expect(f.sandbox.modal_object_id).toBe("sb-real"); + expect(f.store.read()?.phase).toBe("running"); + expect(f.store.read()?.restoreInvoked).toBeUndefined(); + expect(pending).toBe(formatPendingVmReference("test-session", generation.sandboxId)); + }); + + it("stops resolving when the row is fenced mid-loop", async () => { + vi.useFakeTimers(); + const f = fixture(); + f.client.resolveVmSandbox.mockRejectedValue(new ModalApiError("invisible", 409, "not_visible")); + const spawning = f.makeManager().spawnSandbox(); + await vi.waitFor(() => expect(f.client.resolveVmSandbox).toHaveBeenCalledOnce()); + f.sandbox.status = "failed"; + f.sandbox.fenced = 1; + await vi.advanceTimersByTimeAsync(20_000); + await spawning; + expect(f.client.resolveVmSandbox).toHaveBeenCalledOnce(); + expect(f.sandbox.modal_object_id).not.toBe("sb-real"); + }); + + it("destroys a result that arrives after the row is fenced", async () => { + const f = fixture(); + let resolve!: (value: ResolveVmSandboxResponse) => void; + f.client.resolveVmSandbox.mockImplementationOnce(() => new Promise((done) => (resolve = done))); + const spawning = f.makeManager().spawnSandbox(); + await vi.waitFor(() => expect(f.client.resolveVmSandbox).toHaveBeenCalledOnce()); + f.sandbox.status = "failed"; + f.sandbox.fenced = 1; + resolve({ + sandboxId: f.sandbox.modal_sandbox_id!, + modalObjectId: "sb-late", + sandboxBackend: "modal-vm", + }); + await spawning; + expect(f.client.stopSandbox).toHaveBeenCalledWith( + expect.objectContaining({ providerObjectId: "sb-late" }), + undefined + ); + expect(f.sandbox.modal_object_id).not.toBe("sb-late"); + }); + + it("reconciles a restarted bridge without blocking readiness or writing a replaced generation", async () => { + const f = fixture(); + const original = f.makeManager(); + f.client.createSandbox.mockImplementationOnce(() => new Promise(() => {})); + void original.spawnSandbox(); + await vi.waitFor(() => expect(f.client.createSandbox).toHaveBeenCalledOnce()); + const generation = { sandboxId: f.sandbox.modal_sandbox_id!, createdAt: f.sandbox.created_at }; + expect(f.sandbox.modal_object_id).toBe( + formatPendingVmReference("test-session", generation.sandboxId) + ); + let resolve!: (value: Awaited>) => void; + f.client.resolveVmSandbox.mockImplementationOnce(() => new Promise((done) => (resolve = done))); + const restarted = f.makeManager(); + restarted.onSandboxSocketAttached(generation); + expect(f.backgroundTasks.submit).toHaveBeenCalledWith(expect.any(Function), { + name: "sandbox.vm_resolve", + }); + expect(restarted.onRuntimeReady(Date.now(), undefined, 1)).toBe(true); + await vi.waitFor(() => expect(f.client.resolveVmSandbox).toHaveBeenCalledOnce()); + expect(f.sandbox.modal_object_id).not.toBe("sb-real"); + expect(f.client.resolveVmSandbox).toHaveBeenCalledOnce(); + resolve({ + sandboxId: generation.sandboxId, + modalObjectId: "sb-real", + sandboxBackend: "modal-vm", + codeServerUrl: "https://editor.example", + codeServerPassword: "password", + }); + await vi.waitFor(() => expect(f.sandbox.modal_object_id).toBe("sb-real")); + expect(f.store.read()?.providerObjectId).toBe("sb-real"); + expect(f.store.read()?.expiresAtMs).toBe(generation.createdAt + 3_600_000); + expect(f.sandbox.ttyd_token).toBeNull(); + }); + + it("mints terminal access when the bridge resolves while the original instance holds the token", async () => { + const f = fixture(); + f.client.createSandbox.mockImplementationOnce(() => new Promise(() => {})); + const manager = f.makeManager(); + void manager.spawnSandbox(); + await vi.waitFor(() => expect(f.client.createSandbox).toHaveBeenCalledOnce()); + manager.onSandboxSocketAttached({ + sandboxId: f.sandbox.modal_sandbox_id!, + createdAt: f.sandbox.created_at, + }); + await vi.waitFor(() => expect(f.sandbox.modal_object_id).toBe("sb-real")); + expect(f.sandbox.ttyd_url).toBe("https://terminal.example"); + expect(f.sandbox.ttyd_token).toBeTruthy(); + }); + + it("claims a restore when the bridge resolved its handle before the lost response", async () => { + const f = fixture("restore"); + let rejectRestore!: (error: Error) => void; + f.client.restoreSandbox.mockImplementationOnce( + () => new Promise((_resolve, reject) => (rejectRestore = reject)) + ); + const manager = f.makeManager(); + const restoring = manager.spawnSandbox(); + await vi.waitFor(() => expect(f.client.restoreSandbox).toHaveBeenCalledOnce()); + const generation = { sandboxId: f.sandbox.modal_sandbox_id!, createdAt: f.sandbox.created_at }; + manager.onSandboxSocketAttached(generation); + await vi.waitFor(() => expect(f.sandbox.modal_object_id).toBe("sb-real")); + expect(f.store.read()).toMatchObject({ phase: "restoring", restoreInvoked: true }); + + expect(manager.onRuntimeReady(Date.now(), undefined, 1)).toBe(true); + f.sandbox.status = "ready"; + manager.onShutdownGenerationReady({ + type: "sandbox_generation_ready", + generation, + sandboxId: generation.sandboxId, + timestamp: Date.now(), + }); + rejectRestore(new ModalApiError("pending race", 409, "race_pending")); + await restoring; + + expect(f.client.resolveVmSandbox).toHaveBeenCalledOnce(); + expect(f.store.read()).toMatchObject({ phase: "running", providerObjectId: "sb-real" }); + expect(f.store.read()?.restoreInvoked).toBeUndefined(); + expect(manager.pushAdmissionDecision()).toBe("ready"); + expect(f.sandbox.modal_object_id).toBe("sb-real"); + }); + + it("does not attach bridge access to a newer generation", async () => { + const f = fixture(); + f.client.createSandbox.mockImplementationOnce(() => new Promise(() => {})); + void f.makeManager().spawnSandbox(); + await vi.waitFor(() => expect(f.client.createSandbox).toHaveBeenCalledOnce()); + let resolve!: (value: ResolveVmSandboxResponse) => void; + f.client.resolveVmSandbox.mockImplementationOnce(() => new Promise((done) => (resolve = done))); + const restarted = f.makeManager(); + restarted.onSandboxSocketAttached({ + sandboxId: f.sandbox.modal_sandbox_id!, + createdAt: f.sandbox.created_at, + }); + await vi.waitFor(() => expect(f.client.resolveVmSandbox).toHaveBeenCalledOnce()); + const oldSandboxId = f.sandbox.modal_sandbox_id!; + f.sandbox.modal_sandbox_id = "newer-generation"; + f.sandbox.created_at += 1; + f.sandbox.modal_object_id = "sb-newer"; + resolve({ sandboxId: oldSandboxId, modalObjectId: "sb-real", sandboxBackend: "modal-vm" }); + await vi.waitFor(() => expect(f.storage.completeProviderResume).toHaveBeenCalledOnce()); + expect(f.sandbox.modal_object_id).toBe("sb-newer"); + expect(f.store.read()?.providerObjectId).not.toBe("sb-real"); + }); + + it("retries transient bridge lookups after readiness until the same generation resolves", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2030-01-01T00:00:00Z")); + const f = fixture(); + f.client.createSandbox.mockImplementationOnce(() => new Promise(() => {})); + void f.makeManager().spawnSandbox(); + await vi.waitFor(() => expect(f.client.createSandbox).toHaveBeenCalledOnce()); + f.client.resolveVmSandbox + .mockRejectedValueOnce(new ModalApiError("invisible", 409, "not_visible")) + .mockRejectedValueOnce(new ModalApiError("unavailable", 503)); + const restarted = f.makeManager(); + const generation = { sandboxId: f.sandbox.modal_sandbox_id!, createdAt: f.sandbox.created_at }; + restarted.onSandboxSocketAttached(generation); + expect(restarted.onRuntimeReady(Date.now(), undefined, 1)).toBe(true); + await vi.advanceTimersByTimeAsync(25_000); + expect(f.client.resolveVmSandbox).toHaveBeenCalledTimes(3); + expect(f.sandbox.modal_object_id).toBe("sb-real"); + }); + + it("retries a transient bridge error even when the generation predates the visibility bound", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2030-01-01T00:00:00Z")); + const f = fixture(); + f.sandbox.status = "ready"; + f.sandbox.created_at = Date.now() - PENDING_VM_REFERENCE_MATERIALIZE_BOUND_MS - 1; + f.sandbox.modal_object_id = formatPendingVmReference( + "test-session", + f.sandbox.modal_sandbox_id! + ); + f.client.resolveVmSandbox.mockRejectedValueOnce(new ModalApiError("unavailable", 503)); + f.makeManager().onSandboxSocketAttached({ + sandboxId: f.sandbox.modal_sandbox_id!, + createdAt: f.sandbox.created_at, + }); + await vi.advanceTimersByTimeAsync(15_000); + expect(f.client.resolveVmSandbox).toHaveBeenCalledTimes(2); + expect(f.sandbox.modal_object_id).toBe("sb-real"); + }); + + it("stops bridge retries after the bounded window while leaving the pending handle intact", async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date("2030-01-01T00:00:00Z")); + const f = fixture(); + f.client.createSandbox.mockImplementationOnce(() => new Promise(() => {})); + void f.makeManager().spawnSandbox(); + await vi.waitFor(() => expect(f.client.createSandbox).toHaveBeenCalledOnce()); + f.client.resolveVmSandbox.mockRejectedValue(new ModalApiError("unavailable", 503)); + const restarted = f.makeManager(); + restarted.onSandboxSocketAttached({ + sandboxId: f.sandbox.modal_sandbox_id!, + createdAt: f.sandbox.created_at, + }); + expect(restarted.onRuntimeReady(Date.now(), undefined, 1)).toBe(true); + await vi.advanceTimersByTimeAsync(PENDING_VM_REFERENCE_MATERIALIZE_BOUND_MS + 20_000); + const attempts = f.client.resolveVmSandbox.mock.calls.length; + await vi.advanceTimersByTimeAsync(30_000); + expect(f.client.resolveVmSandbox).toHaveBeenCalledTimes(attempts); + expect(f.sandbox.modal_object_id).toBe( + formatPendingVmReference("test-session", f.sandbox.modal_sandbox_id!) + ); + }); + + it("retries the newer generation if its bridge attaches during an older lookup", async () => { + const f = fixture(); + f.client.createSandbox.mockImplementationOnce(() => new Promise(() => {})); + void f.makeManager().spawnSandbox(); + await vi.waitFor(() => expect(f.client.createSandbox).toHaveBeenCalledOnce()); + const restarted = f.makeManager(); + let rejectFirst!: (error: Error) => void; + f.client.resolveVmSandbox.mockImplementationOnce( + () => new Promise((_resolve, reject) => (rejectFirst = reject)) + ); + restarted.onSandboxSocketAttached({ + sandboxId: f.sandbox.modal_sandbox_id!, + createdAt: f.sandbox.created_at, + }); + await vi.waitFor(() => expect(f.client.resolveVmSandbox).toHaveBeenCalledOnce()); + const newer = { sandboxId: "newer-generation", createdAt: f.sandbox.created_at + 1 }; + f.sandbox.modal_sandbox_id = newer.sandboxId; + f.sandbox.created_at = newer.createdAt; + f.sandbox.modal_object_id = formatPendingVmReference("test-session", newer.sandboxId); + f.sandbox.status = "connecting"; + restarted.onSandboxSocketAttached(newer); + restarted.onRuntimeReady(Date.now(), undefined, 1); + rejectFirst(new ModalApiError("invisible", 409, "not_visible")); + await vi.waitFor(() => expect(f.client.resolveVmSandbox).toHaveBeenCalledTimes(2)); + expect(f.client.resolveVmSandbox.mock.calls[1][0]).toMatchObject({ + sandboxId: newer.sandboxId, + }); + await vi.waitFor(() => expect(f.sandbox.modal_object_id).toBe("sb-real")); + }); +}); diff --git a/packages/control-plane/src/sandbox/provider.ts b/packages/control-plane/src/sandbox/provider.ts index 9c9e78d447..b4f019a90e 100644 --- a/packages/control-plane/src/sandbox/provider.ts +++ b/packages/control-plane/src/sandbox/provider.ts @@ -202,6 +202,15 @@ export interface CreateSandboxResult { tunnelUrls?: Record; } +export interface ResolveSandboxConfig { + sessionId: string; + sandboxId: string; + generationCreatedAtMs: number; + timeoutSeconds?: number; +} + +export type ResolveSandboxResult = Omit; + /** * Configuration for restoring a sandbox from a snapshot. */ @@ -587,6 +596,12 @@ export interface SandboxProvider { > ): PendingSandboxAllocation | undefined; + /** Lookup only, for a VM launch whose response was lost. */ + resolveSandbox?(config: ResolveSandboxConfig): Promise; + + /** Whether a failed launch could still have created this generation. */ + isUnknownStartupError?(error: unknown): boolean; + /** * Create a new sandbox. * diff --git a/packages/control-plane/src/sandbox/providers/modal-provider.test.ts b/packages/control-plane/src/sandbox/providers/modal-provider.test.ts index 1cf2b77f55..83ec94cb20 100644 --- a/packages/control-plane/src/sandbox/providers/modal-provider.test.ts +++ b/packages/control-plane/src/sandbox/providers/modal-provider.test.ts @@ -94,6 +94,39 @@ const testConfig = { // ==================== Tests ==================== describe("ModalSandboxProvider", () => { + it.each([ + [409, "race_pending", true], + [409, "other_generation", false], + [409, "window_closed", false], + [502, undefined, true], + [500, undefined, true], + ] as const)( + "classifies VM launch HTTP %s / %s without matching messages", + async (status, detail, unknown) => { + const error = new ModalApiError("arbitrary message", status, detail); + const provider = new ModalSandboxProvider( + createMockModalClient({ + createSandbox: async () => { + throw error; + }, + }), + "modal-vm" + ); + let caught: unknown; + try { + await provider.createSandbox({ ...testConfig, generationCreatedAtMs: Date.now() }); + } catch (cause) { + caught = cause; + } + expect(caught).toBeInstanceOf(SandboxProviderError); + expect((caught as SandboxProviderError).cause).toBe(error); + expect(provider.isUnknownStartupError(caught)).toBe(unknown); + expect((caught as SandboxProviderError).errorType).toBe( + detail === "other_generation" ? "permanent" : "transient" + ); + } + ); + it("derives a pending VM reference and lifetime from the launch reservation", async () => { vi.useFakeTimers({ toFake: ["Date"] }); try { diff --git a/packages/control-plane/src/sandbox/providers/modal-provider.ts b/packages/control-plane/src/sandbox/providers/modal-provider.ts index 5e203e8ffe..957a1902f6 100644 --- a/packages/control-plane/src/sandbox/providers/modal-provider.ts +++ b/packages/control-plane/src/sandbox/providers/modal-provider.ts @@ -5,7 +5,7 @@ * enabling unit testing and future provider abstraction. */ -import { ModalApiError } from "../client"; +import { ModalApiError, ModalVmStartupError } from "../client"; import { formatPendingVmReference, parsePendingVmReference } from "./pending-vm-reference"; import { PENDING_VM_REFERENCE_LAUNCH_WINDOW_MS, @@ -31,6 +31,8 @@ import { type CreateSandboxResult, type RestoreConfig, type RestoreResult, + type ResolveSandboxConfig, + type ResolveSandboxResult, type SnapshotConfig, type SnapshotResult, type StopConfig, @@ -115,6 +117,45 @@ export class ModalSandboxProvider implements SandboxProvider, ModalImageBuildPro }; } + isUnknownStartupError(error: unknown): boolean { + if (this.name !== "modal-vm") return false; + const cause = error instanceof SandboxProviderError ? error.cause : error; + if (cause instanceof ModalVmStartupError) + return cause.outcome === "unknown" || cause.outcome === "race_pending"; + if (cause instanceof ModalApiError) + return cause.detail === "race_pending" || cause.status >= 500; + return cause instanceof TypeError || SandboxProviderError.isTransientNetworkError(cause); + } + + async resolveSandbox(config: ResolveSandboxConfig): Promise { + if (this.name !== "modal-vm") + throw new SandboxProviderError("VM resolution requires modal-vm", "permanent"); + try { + const result = await this.client.resolveVmSandbox({ + sessionId: config.sessionId, + sandboxId: config.sandboxId, + }); + this.confirmSessionLaunch(result); + if (result.sandboxId !== config.sandboxId || !result.modalObjectId) + throw new SandboxProviderError( + "Modal VM resolution returned a different generation", + "permanent" + ); + return { + sandboxId: result.sandboxId, + providerObjectId: result.modalObjectId, + lifetime: this.launchLifetime(config), + codeServerUrl: result.codeServerUrl, + codeServerPassword: result.codeServerPassword, + vncAccess: createVncAccess(result.vncUrl, result.vncPassword), + ttydUrl: result.ttydUrl, + tunnelUrls: result.tunnelUrls, + }; + } catch (error) { + throw this.classifyError("Failed to resolve Modal VM", error); + } + } + private launchLifetime( config: Pick, observedAtMs?: number @@ -262,15 +303,6 @@ export class ModalSandboxProvider implements SandboxProvider, ModalImageBuildPro tunnelUrls: result.tunnelUrls, }; } catch (error) { - if (error instanceof ModalApiError) { - throw this.classifyErrorWithStatus( - `Restore failed with HTTP ${error.status}`, - error.status - ); - } - if (error instanceof SandboxProviderError) { - throw error; - } throw this.classifyError("Failed to restore sandbox from snapshot", error); } } @@ -520,6 +552,30 @@ export class ModalSandboxProvider implements SandboxProvider, ModalImageBuildPro */ private classifyError(message: string, error: unknown): SandboxProviderError { if (error instanceof SandboxProviderError) return error; + if (error instanceof ModalVmStartupError) + return new SandboxProviderError( + `${message}: ${error.message}`, + error.outcome === "other_generation" ? "permanent" : "transient", + error + ); + if (error instanceof ModalApiError) { + const context = `${message} with HTTP ${error.status}`; + if (this.name === "modal-vm") { + if ( + error.detail === "not_visible" || + error.detail === "window_closed" || + error.detail === "race_pending" || + error.detail === "other_generation" + ) + return new SandboxProviderError( + context, + error.detail === "other_generation" ? "permanent" : "transient", + error + ); + if (error.status >= 500) return new SandboxProviderError(context, "transient", error); + } + return this.classifyErrorWithStatus(context, error.status, error); + } if (SandboxProviderError.isTransientNetworkError(error)) { return new SandboxProviderError( `${message}: ${error instanceof Error ? error.message : String(error)}`, diff --git a/packages/control-plane/src/session/components.ts b/packages/control-plane/src/session/components.ts index 1eb6a2ae9e..1723e34300 100644 --- a/packages/control-plane/src/session/components.ts +++ b/packages/control-plane/src/session/components.ts @@ -59,6 +59,7 @@ import { requireRepoSecretsEncryptionKey, requireTokenEncryptionKey } from "../e import type { Env, ClientInfo } from "../types"; import type { SessionRow } from "./types"; import type { SqlDatabase } from "../db/sql-database"; +import type { BackgroundTasks } from "../platform-ports"; import type { SessionPlatform } from "./platform"; import { SessionCoreRepository } from "./session-core-repository"; // The composition root grants each consumer only its declared sandbox port. @@ -469,6 +470,7 @@ export function createSessionRuntime(platform: SessionPlatform, env: Env): Sessi wsManager, alarmScheduler, sandboxDashboardSettings, + backgroundTasks, recordWarning: (message, eventId) => recordSessionWarning(eventRepository, messenger, message, eventId), }); @@ -1011,6 +1013,7 @@ interface LifecycleManagerDeps { wsManager: SessionWebSocketManager; alarmScheduler: RehydratableAlarmScheduler; sandboxDashboardSettings: SandboxDashboardSettings; + backgroundTasks: BackgroundTasks; } /** Create the lifecycle manager with all required adapters. */ @@ -1028,6 +1031,7 @@ function createLifecycleManager(deps: LifecycleManagerDeps): SandboxLifecycleMan wsManager, alarmScheduler, sandboxDashboardSettings, + backgroundTasks, } = deps; // Both throw on a misconfigured deployment — deliberately at graph // construction, so every session request fails at initialization instead of @@ -1131,6 +1135,7 @@ function createLifecycleManager(deps: LifecycleManagerDeps): SandboxLifecycleMan idGenerator, shutdown, config, - imageBuildLookup + imageBuildLookup, + backgroundTasks ); } diff --git a/packages/control-plane/src/session/sandbox-repository.test.ts b/packages/control-plane/src/session/sandbox-repository.test.ts index 08914294b4..cf8cb05704 100644 --- a/packages/control-plane/src/session/sandbox-repository.test.ts +++ b/packages/control-plane/src/session/sandbox-repository.test.ts @@ -575,6 +575,30 @@ describe("SandboxRepository boot state (SQLite)", () => { }); }); + it("resolves a VM only while its generation and pending handle still match", async () => { + const { repository, set } = createSqliteRepository(); + const generation = { sandboxId: "sb-1", createdAt: 1000 }; + const access = { + providerObjectId: "sb-real", + codeServer: { url: "https://editor.example", password: "secret" }, + vnc: null, + ttyd: null, + tunnelUrls: { "8080": "https://port.example" }, + }; + set("status = 'connecting', modal_sandbox_id = 'sb-1', modal_object_id = 'pending'"); + expect(await repository.completeProviderResume(generation, access, "other")).toBe(false); + expect(repository.getSandbox()?.modal_object_id).toBe("pending"); + expect(await repository.completeProviderResume(generation, access, "pending")).toBe(true); + expect(repository.getSandbox()).toMatchObject({ + modal_object_id: "sb-real", + code_server_url: "https://editor.example", + }); + expect(await repository.getSandboxAccessSecret("codeServer")).toBe("secret"); + set("modal_sandbox_id = 'sb-2', created_at = 2000, modal_object_id = 'pending'"); + expect(await repository.completeProviderResume(generation, access, "pending")).toBe(false); + expect(repository.getSandbox()?.modal_object_id).toBe("pending"); + }); + describe("markSandboxReady", () => { const generation = { sandboxId: "sb-1", createdAt: 1000 }; diff --git a/packages/control-plane/src/session/sandbox-repository.ts b/packages/control-plane/src/session/sandbox-repository.ts index 987a38422e..fcc392c7b9 100644 --- a/packages/control-plane/src/session/sandbox-repository.ts +++ b/packages/control-plane/src/session/sandbox-repository.ts @@ -382,7 +382,8 @@ export class SandboxRepository { */ async completeProviderResume( generation: { sandboxId: string | null; createdAt: number }, - access: ProviderResumeAccessData + access: ProviderResumeAccessData, + expectedProviderObjectId?: string ): Promise { const [codeServerPassword, vncPassword, ttydToken] = await Promise.all([ access.codeServer ? this.encrypt(access.codeServer.password) : null, @@ -401,7 +402,8 @@ export class SandboxRepository { tunnel_urls = ? WHERE id = (SELECT id FROM sandbox LIMIT 1) AND modal_sandbox_id IS ? AND created_at = ? - AND status IN ('connecting', 'ready') AND fenced = 0`, + AND (status IN ('connecting', 'ready') OR (? IS NOT NULL AND status = 'spawning')) + AND fenced = 0 AND (? IS NULL OR modal_object_id = ?)`, access.providerObjectId, access.codeServer?.url ?? null, codeServerPassword, @@ -411,7 +413,10 @@ export class SandboxRepository { ttydToken, access.tunnelUrls ? JSON.stringify(access.tunnelUrls) : null, generation.sandboxId, - generation.createdAt + generation.createdAt, + expectedProviderObjectId ?? null, + expectedProviderObjectId ?? null, + expectedProviderObjectId ?? null ); result.toArray(); return (result.rowsWritten ?? 0) > 0; diff --git a/packages/control-plane/src/session/sandbox-shutdown.ts b/packages/control-plane/src/session/sandbox-shutdown.ts index 8eeaba5754..f76bbc65b3 100644 --- a/packages/control-plane/src/session/sandbox-shutdown.ts +++ b/packages/control-plane/src/session/sandbox-shutdown.ts @@ -295,6 +295,25 @@ export class SandboxShutdownCoordinator { this.notifyLifecycleChange(); } + /** Bridge recovery replaces only the pending handle, leaving the lifetime and restore hold intact. */ + recordResolvedProviderHandle( + generation: SandboxGeneration, + expectedReference: string, + providerObjectId: string + ): void { + const state = this.deps.store.read(); + const row = this.deps.sandbox.getSandbox(); + if ( + !state || + !this.current(state) || + !this.matches(state, generation) || + state.providerObjectId !== expectedReference || + row?.modal_object_id !== providerObjectId + ) + return; + this.publish({ ...state, providerObjectId }); + } + runtimeReady(version?: 1): void { const state = this.deps.store.read(); if (!state || !this.current(state)) return; diff --git a/packages/modal-infra/src/sandbox/launch.py b/packages/modal-infra/src/sandbox/launch.py index 7aee1fbb04..e3dde208ea 100644 --- a/packages/modal-infra/src/sandbox/launch.py +++ b/packages/modal-infra/src/sandbox/launch.py @@ -9,9 +9,12 @@ import modal from sandbox_runtime.constants import ( + CODE_SERVER_PORT_ENV_VAR, DOCKER_ENABLED_ENV_VAR, + EXPECTED_TUNNEL_PORTS_ENV_VAR, NOVNC_PORT_ENV_VAR, SANDBOX_TIMEOUT_ENV_VAR, + TTYD_PROXY_PORT_ENV_VAR, VNC_PASSWORD_ENV_VAR, VNC_PASSWORD_MAX_BYTES, ) @@ -32,6 +35,7 @@ from .models import SandboxConfig, SandboxHandle from .tunnels import SandboxTunnels from .vcs_env import inject_vcs_env_vars +from .vm_recovery import VMAllocationOutcome, VMServiceLaunch, find_owned_vm, owned_vm_tags_match _RESERVED_LAUNCH_ENV_VARS = { "RESTORED_FROM_SNAPSHOT", @@ -41,6 +45,10 @@ "TERMINAL_ENABLED", "AGENT_SLACK_NOTIFY_ENABLED", "SESSION_CONFIG", + "CODE_SERVER_PASSWORD", + CODE_SERVER_PORT_ENV_VAR, + TTYD_PROXY_PORT_ENV_VAR, + EXPECTED_TUNNEL_PORTS_ENV_VAR, VNC_PASSWORD_ENV_VAR, NOVNC_PORT_ENV_VAR, DOCKER_ENABLED_ENV_VAR, @@ -225,6 +233,7 @@ async def launch(self, spec: SandboxLaunchSpec) -> SandboxHandle: create_kwargs=create_kwargs, repository_image=repository_image, launch_deadline_at_ms=config.launch_deadline_at_ms, + service_launch=VMServiceLaunch.from_tunnels(tunnels), ) if adopted: passwords = await self._read_access_passwords( @@ -263,6 +272,7 @@ async def _launch_docker_sandbox( retire_sandbox_id: str | None, create_kwargs: dict[str, Any], repository_image: bool, + service_launch: VMServiceLaunch, launch_deadline_at_ms: int | None = None, ) -> tuple[modal.Sandbox, bool]: """Create a named VM or adopt only the allocation owned by this generation.""" @@ -273,17 +283,19 @@ async def _launch_docker_sandbox( existing = await self._find_owned_docker_allocation(name, tags) if existing is None: if launch_deadline_at_ms is not None and time.time() * 1000 >= launch_deadline_at_ms: - raise RuntimeError("VM launch deadline expired") + raise VMAllocationOutcome("window_closed", "VM launch deadline expired") try: sandbox = await _create_sandbox( - {**create_kwargs, "name": name, "tags": tags}, + {**create_kwargs, "name": name, "tags": {**tags, **service_launch.tags()}}, repository_image=repository_image, ) return sandbox, False - except modal.exception.AlreadyExistsError: + except modal.exception.AlreadyExistsError as e: existing = await self._find_owned_docker_allocation(name, tags) if existing is None: - raise + raise VMAllocationOutcome( + "race_pending", "VM allocation is not yet visible" + ) from e log.info( "sandbox.docker_allocation_adopted", sandbox_id=sandbox_id, @@ -328,13 +340,8 @@ async def _read_access_passwords( async def _find_owned_docker_allocation( name: str, tags: dict[str, str] ) -> modal.Sandbox | None: - try: - sandbox = await modal.Sandbox.from_name.aio(APP_NAME, name) - except modal.exception.NotFoundError: - return None - if await sandbox.get_tags.aio() != tags: - raise RuntimeError("Docker sandbox allocation ownership mismatch") - return sandbox + found = await find_owned_vm(name, tags) + return found[0] if found else None async def _retire_docker_allocation(self, session_id: str, sandbox_id: str) -> None: """Terminate a prior named VM only when its ownership tags match.""" @@ -343,7 +350,9 @@ async def _retire_docker_allocation(self, session_id: str, sandbox_id: str) -> N sandbox = await modal.Sandbox.from_name.aio(APP_NAME, name) except modal.exception.NotFoundError: return - if await sandbox.get_tags.aio() != docker_allocation_tags(session_id, sandbox_id): + if not owned_vm_tags_match( + await sandbox.get_tags.aio(), docker_allocation_tags(session_id, sandbox_id) + ): log.warn("sandbox.docker_allocation_retire_mismatch", sandbox_id=sandbox_id) return await sandbox.terminate.aio(wait=True) diff --git a/packages/modal-infra/src/sandbox/manager.py b/packages/modal-infra/src/sandbox/manager.py index 7a82b3f9f8..0f1c4dd46a 100644 --- a/packages/modal-infra/src/sandbox/manager.py +++ b/packages/modal-infra/src/sandbox/manager.py @@ -46,6 +46,12 @@ ) from .models import DEFAULT_VNC_ENABLED, SandboxConfig, SandboxHandle from .tunnels import MAX_TUNNEL_PORTS +from .vm_recovery import ( + VMAllocationOutcome, + find_owned_vm, + owned_vm_tags_match, + recover_vm_access, +) # Preserve the existing public imports after moving their implementations. __all__ = [ @@ -75,6 +81,7 @@ "SandboxConfig", "SandboxHandle", "SandboxManager", + "VMAllocationOutcome", ] log = get_logger("manager") @@ -252,7 +259,9 @@ async def get_sandbox_by_id(self, sandbox_id: str) -> SandboxHandle | None: except modal.exception.NotFoundError: return None tags = await modal_sandbox.get_tags.aio() - if identity is not None and tags != docker_allocation_tags(*identity): + if identity is not None and not owned_vm_tags_match( + tags, docker_allocation_tags(*identity) + ): raise PendingVMReferenceNotVisible("Docker sandbox allocation ownership mismatch") backend = tags.get("openinspect_backend", "modal") if backend not in ("modal", "modal-vm"): @@ -266,6 +275,32 @@ async def get_sandbox_by_id(self, sandbox_id: str) -> SandboxHandle | None: created_at=time.time(), ) + async def resolve_vm_sandbox(self, session_id: str, sandbox_id: str) -> SandboxHandle: + """Recover only the running generation's identity and versioned access metadata.""" + found = await find_owned_vm( + docker_allocation_name(session_id), docker_allocation_tags(session_id, sandbox_id) + ) + if found is None: + raise VMAllocationOutcome("not_visible", "VM allocation is not visible") + sandbox, tags = found + access = await recover_vm_access( + sandbox, sandbox_id, tags, SandboxLauncher._read_access_passwords + ) + return SandboxHandle( + sandbox_id=sandbox_id, + modal_sandbox=sandbox, + status=SandboxStatus.WARMING, + created_at=time.time(), + modal_object_id=sandbox.object_id, + code_server_url=access.code_server_url, + code_server_password=access.code_server_password, + vnc_url=access.vnc_url, + vnc_password=access.vnc_password, + ttyd_url=access.ttyd_url, + tunnel_urls=access.tunnel_urls, + sandbox_backend="modal-vm", + ) + async def restore_from_snapshot( self, snapshot_image_id: str, diff --git a/packages/modal-infra/src/sandbox/tunnels.py b/packages/modal-infra/src/sandbox/tunnels.py index 17210838aa..831854eefb 100644 --- a/packages/modal-infra/src/sandbox/tunnels.py +++ b/packages/modal-infra/src/sandbox/tunnels.py @@ -50,12 +50,16 @@ def __init__( settings: dict[str, Any] | None = None, ) -> None: settings = settings or {} - code_server_port, novnc_port, ttyd_proxy_port = self._resolve_service_ports(settings) + self.service_ports = self._resolve_service_ports(settings) + self.service_enabled = ( + code_server_enabled, + vnc_enabled, + bool(settings.get("terminalEnabled", False)), + ) + code_server_port, novnc_port, ttyd_proxy_port = self.service_ports self._code_server_port = code_server_port if code_server_enabled else None self._novnc_port = novnc_port if vnc_enabled else None - self._ttyd_proxy_port = ( - ttyd_proxy_port if bool(settings.get("terminalEnabled", False)) else None - ) + self._ttyd_proxy_port = ttyd_proxy_port if self.service_enabled[2] else None service_ports = [ port for port in (self._code_server_port, self._novnc_port, self._ttyd_proxy_port) @@ -63,12 +67,12 @@ def __init__( ] reserved = {VNC_PORT, *service_ports} raw_ports = settings.get("tunnelPorts", []) - self._extra_ports = ( + self.extra_ports = ( [port for port in self._validate_ports(raw_ports) if port not in reserved] if raw_ports else [] ) - self.exposed_ports = service_ports + self._extra_ports + self.exposed_ports = service_ports + self.extra_ports @property def environment(self) -> dict[str, str]: @@ -81,11 +85,13 @@ def environment(self) -> dict[str, str]: if self._ttyd_proxy_port is not None: env["TERMINAL_ENABLED"] = "true" env[TTYD_PROXY_PORT_ENV_VAR] = str(self._ttyd_proxy_port) - if self._extra_ports: - env[EXPECTED_TUNNEL_PORTS_ENV_VAR] = ",".join(str(p) for p in self._extra_ports) + if self.extra_ports: + env[EXPECTED_TUNNEL_PORTS_ENV_VAR] = ",".join(str(p) for p in self.extra_ports) return env - async def resolve(self, sandbox: modal.Sandbox, sandbox_id: str) -> TunnelUrls: + async def resolve( + self, sandbox: modal.Sandbox, sandbox_id: str, *, write_env_file: bool = True + ) -> TunnelUrls: """Resolve URLs and publish extras; partial resolution/write failures are non-fatal.""" if not self.exposed_ports: return TunnelUrls() @@ -102,7 +108,7 @@ async def resolve(self, sandbox: modal.Sandbox, sandbox_id: str) -> TunnelUrls: resolved.pop(self._ttyd_proxy_port, None) if self._ttyd_proxy_port is not None else None ) extra_urls = resolved or None - if extra_urls: + if extra_urls and write_env_file: await self._write_tunnel_env_file(sandbox, sandbox_id, extra_urls) return TunnelUrls( code_server_url=code_server_url, diff --git a/packages/modal-infra/src/sandbox/vm_recovery.py b/packages/modal-infra/src/sandbox/vm_recovery.py new file mode 100644 index 0000000000..3f6759df15 --- /dev/null +++ b/packages/modal-infra/src/sandbox/vm_recovery.py @@ -0,0 +1,157 @@ +"""Lookup-only recovery of named VM allocations and their provider-owned access metadata.""" + +from collections.abc import Awaitable, Callable +from dataclasses import dataclass +from typing import Literal + +import modal + +from sandbox_runtime.constants import VNC_PASSWORD_ENV_VAR + +from ..app_config import APP_NAME +from .tunnels import SandboxTunnels + +VM_LAUNCH_TAG = "openinspect_vm_launch" +VM_PORTS_TAG = "openinspect_vm_ports" +_METADATA_TAGS = {VM_LAUNCH_TAG, VM_PORTS_TAG} + +type VMAllocationDetail = Literal[ + "not_visible", "other_generation", "window_closed", "race_pending" +] + + +class VMAllocationOutcome(RuntimeError): + """A known named-VM lookup or launch outcome, distinct from provider failures.""" + + def __init__(self, detail: VMAllocationDetail, message: str): + super().__init__(message) + self.detail = detail + + +@dataclass(frozen=True) +class VMServiceLaunch: + code_server_enabled: bool + vnc_enabled: bool + terminal_enabled: bool + code_server_port: int + novnc_port: int + ttyd_proxy_port: int + tunnel_ports: list[int] + + @classmethod + def from_tunnels(cls, tunnels: SandboxTunnels) -> "VMServiceLaunch": + return cls(*tunnels.service_enabled, *tunnels.service_ports, tunnels.extra_ports) + + def tags(self) -> dict[str, str]: + # Two short, tag-safe values keep even ten five-digit extra ports within Modal's limit. + flags = "".join( + "1" if enabled else "0" + for enabled in (self.code_server_enabled, self.vnc_enabled, self.terminal_enabled) + ) + return { + VM_LAUNCH_TAG: ( + f"1-{flags}-{self.code_server_port}-{self.novnc_port}-{self.ttyd_proxy_port}" + ), + VM_PORTS_TAG: "-".join(map(str, self.tunnel_ports)) or "none", + } + + +def owned_vm_tags_match(actual: dict[str, str], expected: dict[str, str]) -> bool: + """Keep exact generation ownership, allowing only the two launch metadata tags.""" + return all(actual.get(key) == value for key, value in expected.items()) and ( + actual.keys() <= expected.keys() | _METADATA_TAGS + ) + + +def parse_vm_service_launch(tags: dict[str, str]) -> VMServiceLaunch | None: + """Absent, unsupported, or incomplete metadata grants no access (including on legacy VMs).""" + launch = tags.get(VM_LAUNCH_TAG, "").split("-") + raw_ports = tags.get(VM_PORTS_TAG) + if len(launch) != 5 or launch[0] != "1" or len(launch[1]) != 3: + return None + if any(flag not in "01" for flag in launch[1]) or raw_ports is None: + return None + + def port(value: str) -> int | None: + if not value.isascii() or not value.isdecimal() or len(value) > 5: + return None + number = int(value) + return number if 1 <= number <= 65535 else None + + service_ports = [port(value) for value in launch[2:]] + extras = [] if raw_ports == "none" else [port(value) for value in raw_ports.split("-")] + if None in service_ports or None in extras or len(extras) > 10: + return None + return VMServiceLaunch( + *(flag == "1" for flag in launch[1]), + *service_ports, + extras, + ) + + +async def find_owned_vm( + name: str, expected_tags: dict[str, str] +) -> tuple[modal.Sandbox, dict[str, str]] | None: + try: + sandbox = await modal.Sandbox.from_name.aio(APP_NAME, name) + except modal.exception.NotFoundError: + return None + tags = await sandbox.get_tags.aio() + if not owned_vm_tags_match(tags, expected_tags): + raise VMAllocationOutcome( + "other_generation", "Docker sandbox allocation ownership mismatch" + ) + return sandbox, tags + + +@dataclass +class VMAccess: + code_server_url: str | None = None + code_server_password: str | None = None + vnc_url: str | None = None + vnc_password: str | None = None + ttyd_url: str | None = None + tunnel_urls: dict[int, str] | None = None + + +async def recover_vm_access( + sandbox: modal.Sandbox, + sandbox_id: str, + tags: dict[str, str], + read_passwords: Callable[..., Awaitable[dict[str, str]]], +) -> VMAccess: + launch = parse_vm_service_launch(tags) + if launch is None: + return VMAccess() + passwords = await read_passwords( + sandbox, + code_server_enabled=launch.code_server_enabled, + vnc_enabled=launch.vnc_enabled, + ) + tunnels = SandboxTunnels( + code_server_enabled=launch.code_server_enabled, + vnc_enabled=launch.vnc_enabled, + settings={ + "terminalEnabled": launch.terminal_enabled, + "codeServerPort": launch.code_server_port, + "vncPort": launch.novnc_port, + "terminalPort": launch.ttyd_proxy_port, + "tunnelPorts": launch.tunnel_ports, + }, + ) + urls = await tunnels.resolve(sandbox, sandbox_id, write_env_file=False) + if ( + (launch.code_server_enabled and not urls.code_server_url) + or (launch.vnc_enabled and not urls.vnc_url) + or (launch.terminal_enabled and not urls.ttyd_url) + or any(not (urls.tunnel_urls or {}).get(port) for port in launch.tunnel_ports) + ): + raise VMAllocationOutcome("race_pending", "VM allocation tunnels are not yet visible") + return VMAccess( + code_server_url=urls.code_server_url, + code_server_password=passwords.get("CODE_SERVER_PASSWORD"), + vnc_url=urls.vnc_url, + vnc_password=passwords.get(VNC_PASSWORD_ENV_VAR), + ttyd_url=urls.ttyd_url, + tunnel_urls=urls.tunnel_urls, + ) diff --git a/packages/modal-infra/src/web_api.py b/packages/modal-infra/src/web_api.py index 73d76b3acb..16e3677a52 100644 --- a/packages/modal-infra/src/web_api.py +++ b/packages/modal-infra/src/web_api.py @@ -42,6 +42,7 @@ InvalidDockerSettingsError, ModalBackend, ) +from .sandbox.manager import VMAllocationOutcome configure_logging() log = get_logger("web_api") @@ -184,6 +185,13 @@ class RestoreSandboxRequest(_ModalRequestModel): launch_deadline_at_ms: int | None = Field(default=None, gt=0) +class ResolveVMSandboxRequest(_ModalRequestModel): + model_config = ConfigDict(extra="forbid", strict=True) + + session_id: NonEmptyString + sandbox_id: NonEmptyString + + @dataclass class _EndpointExecution: endpoint_name: str @@ -232,6 +240,10 @@ async def _execute_endpoint( execution.http_status = 501 execution.outcome = "error" raise HTTPException(status_code=501, detail="docker_not_available") from e + except VMAllocationOutcome as e: + execution.http_status = 409 + execution.outcome = "error" + raise HTTPException(status_code=execution.http_status, detail=e.detail) from e except Exception as e: execution.http_status = 500 execution.outcome = "error" @@ -286,6 +298,8 @@ def _parse_request[RequestModelT: BaseModel]( }.get(error_type, "user_env_vars has an invalid value") elif field == "timeout_seconds": detail = "timeout_seconds must be a positive integer" + elif error_type == "extra_forbidden": + detail = f"{field} is not allowed" elif len(location) > 1: detail = f"{field} has an invalid value" else: @@ -497,6 +511,48 @@ async def api_create_sandbox( } +@app.function(image=function_image, secrets=[internal_api_secret], timeout=150) +@fastapi_endpoint(method="POST") +async def api_resolve_vm_sandbox( + request: dict, + authorization: str | None = Header(None), + x_trace_id: str | None = Header(None), + x_request_id: str | None = Header(None), + x_session_id: str | None = Header(None), + x_sandbox_id: str | None = Header(None), +) -> dict: + """Authenticated lookup-only VM recovery by session and generation; no create or retire. + + POST body: {"session_id": "...", "sandbox_id": "..."}. No secrets or launch settings. + """ + async with _execute_endpoint( + endpoint_name="api_resolve_vm_sandbox", + authorization=authorization, + trace_id=x_trace_id, + request_id=x_request_id, + session_id=x_session_id, + sandbox_id=x_sandbox_id, + ): + parsed = _parse_request(ResolveVMSandboxRequest, request) + from .sandbox.manager import SandboxManager + + handle = await SandboxManager().resolve_vm_sandbox(parsed.session_id, parsed.sandbox_id) + return { + "success": True, + "data": { + "sandbox_id": handle.sandbox_id, + "modal_object_id": handle.modal_object_id, + "code_server_url": handle.code_server_url, + "code_server_password": handle.code_server_password, + "vnc_url": handle.vnc_url, + "vnc_password": handle.vnc_password, + "ttyd_url": handle.ttyd_url, + "tunnel_urls": handle.tunnel_urls, + "sandbox_backend": handle.sandbox_backend, + }, + } + + @app.function(image=function_image) def deployment_vm_image() -> str | None: """Private SDK-only deployment handshake; preserve capability across selector cutover.""" diff --git a/packages/modal-infra/tests/test_docker_launch.py b/packages/modal-infra/tests/test_docker_launch.py index 1b5aacfac2..36d82b37bb 100644 --- a/packages/modal-infra/tests/test_docker_launch.py +++ b/packages/modal-infra/tests/test_docker_launch.py @@ -18,6 +18,7 @@ parse_launch, parse_pending_vm_reference, ) +from src.sandbox.vm_recovery import VMServiceLaunch, parse_vm_service_launch @pytest.mark.parametrize("settings", [None, {}]) @@ -90,6 +91,15 @@ def test_allocation_tags_bind_session_generation_and_backend(): assert re.fullmatch(r"[a-zA-Z0-9._-]{1,63}", value) +def test_vm_launch_metadata_round_trips_at_max_extra_port_count(): + launch = VMServiceLaunch(True, False, True, 9000, 6080, 7680, list(range(60000, 60010))) + tags = launch.tags() + + assert parse_vm_service_launch(tags) == launch + for value in tags.values(): + assert re.fullmatch(r"[a-zA-Z0-9._-]{1,63}", value) + + def test_pending_vm_reference_uses_shared_two_part_wire_format(): reference = 'modal-vm-session:["session-1","sandbox-1"]' assert parse_pending_vm_reference(reference) == ("session-1", "sandbox-1") diff --git a/packages/modal-infra/tests/test_sandbox_launch.py b/packages/modal-infra/tests/test_sandbox_launch.py index 5831ebbf18..ec83ce45d8 100644 --- a/packages/modal-infra/tests/test_sandbox_launch.py +++ b/packages/modal-infra/tests/test_sandbox_launch.py @@ -32,6 +32,7 @@ SandboxManager, ) from src.sandbox.tunnels import SandboxTunnels, TunnelUrls +from src.sandbox.vm_recovery import VMAllocationOutcome, VMServiceLaunch def _fake_create(captured: dict): @@ -461,11 +462,71 @@ async def test_docker_launch_selects_vm_runtime_and_named_allocation(monkeypatch assert kwargs["cpu"] == (2.0, 2.0) assert kwargs["memory"] == 4096 assert kwargs["name"] == docker_allocation_name("session-1") - assert kwargs["tags"] == docker_allocation_tags("session-1", "sandbox-acme-repo-1700000000000") + assert kwargs["tags"] == { + **docker_allocation_tags("session-1", "sandbox-acme-repo-1700000000000"), + **VMServiceLaunch(False, False, False, 8080, 6080, 7680, []).tags(), + } assert kwargs["env"][DOCKER_ENABLED_ENV_VAR] == "true" assert handle.sandbox_backend == "modal-vm" +@pytest.mark.asyncio +async def test_docker_launch_does_not_allow_user_env_to_spoof_resolved_access(monkeypatch): + manager, captured, _ = _docker_manager(monkeypatch) + monkeypatch.setattr( + "src.sandbox.launch.modal.Sandbox.from_name", + SimpleNamespace(aio=AsyncMock(side_effect=_not_found)), + ) + await manager.create_sandbox( + _docker_config( + user_env_vars={ + "CODE_SERVER_PASSWORD": "spoofed", + VNC_PASSWORD_ENV_VAR: "spoofed", + CODE_SERVER_PORT_ENV_VAR: "9000", + EXPECTED_TUNNEL_PORTS_ENV_VAR: "3000", + } + ) + ) + + for key in ( + "CODE_SERVER_PASSWORD", + VNC_PASSWORD_ENV_VAR, + CODE_SERVER_PORT_ENV_VAR, + EXPECTED_TUNNEL_PORTS_ENV_VAR, + ): + assert key not in captured["kwargs"]["env"] + assert captured["kwargs"]["tags"]["openinspect_vm_launch"] == "1-000-8080-6080-7680" + assert captured["kwargs"]["tags"]["openinspect_vm_ports"] == "none" + + +@pytest.mark.asyncio +async def test_docker_launch_tags_record_effective_enabled_services_and_ports(monkeypatch): + manager, captured, _ = _docker_manager(monkeypatch) + monkeypatch.setattr( + "src.sandbox.launch.modal.Sandbox.from_name", + SimpleNamespace(aio=AsyncMock(side_effect=_not_found)), + ) + + await manager.create_sandbox( + _docker_config( + code_server_enabled=True, + vnc_enabled=True, + settings={ + **DOCKER_SETTINGS, + "terminalEnabled": True, + "codeServerPort": 9000, + "vncPort": 9001, + "terminalPort": 9002, + "tunnelPorts": [3000, 3001], + }, + ) + ) + + assert captured["kwargs"]["tags"]["openinspect_vm_launch"] == "1-111-9000-9001-9002" + assert captured["kwargs"]["tags"]["openinspect_vm_ports"] == "3000-3001" + assert captured["kwargs"]["encrypted_ports"] == [9000, 9001, 9002, 3000, 3001] + + @pytest.mark.asyncio @pytest.mark.parametrize("image_source", ["base", "snapshot"]) async def test_expired_vm_launch_cannot_materialize_after_lookup(monkeypatch, image_source): @@ -486,8 +547,9 @@ async def test_expired_vm_launch_cannot_materialize_after_lookup(monkeypatch, im sandbox_backend="modal-vm", launch_deadline_at_ms=1, ) - with pytest.raises(RuntimeError, match="launch deadline"): + with pytest.raises(VMAllocationOutcome) as exc: await launch + assert exc.value.detail == "window_closed" assert "kwargs" not in captured @@ -638,17 +700,21 @@ async def test_docker_launch_refuses_a_same_named_allocation_it_does_not_own(mon SimpleNamespace(aio=AsyncMock(return_value=foreign)), ) - with pytest.raises(RuntimeError, match="ownership mismatch") as exc: + with pytest.raises(VMAllocationOutcome, match="ownership mismatch") as exc: await manager.create_sandbox(_docker_config()) - assert type(exc.value) is RuntimeError + assert exc.value.detail == "other_generation" assert "kwargs" not in captured @pytest.mark.asyncio async def test_docker_launch_retires_the_prior_generation_only_when_owned(monkeypatch): manager, captured, _ = _docker_manager(monkeypatch) - prior_tags = docker_allocation_tags("session-1", "sandbox-acme-repo-1699999999999") + prior_tags = { + **docker_allocation_tags("session-1", "sandbox-acme-repo-1699999999999"), + "openinspect_vm_launch": "1-000-8080-6080-7680", + "openinspect_vm_ports": "none", + } prior = SimpleNamespace( object_id="modal-prior", get_tags=AsyncMock(return_value=prior_tags), @@ -677,7 +743,7 @@ async def from_name(_app, name): prior.terminate.reset_mock() prior.get_tags = AsyncMock(return_value={"openinspect_kind": "other"}) prior.get_tags.aio = prior.get_tags - with pytest.raises(RuntimeError, match="ownership mismatch"): + with pytest.raises(VMAllocationOutcome, match="ownership mismatch"): await manager.create_sandbox( _docker_config(retire_sandbox_id="sandbox-acme-repo-1699999999999") ) @@ -708,13 +774,14 @@ async def create(kwargs, *, repository_image): return SimpleNamespace(object_id="duplicate-successor") monkeypatch.setattr("src.sandbox.launch._create_sandbox", create) - with pytest.raises(RuntimeError, match="ownership mismatch"): + with pytest.raises(VMAllocationOutcome, match="ownership mismatch"): await launcher._launch_docker_sandbox( session_id="session-1", sandbox_id="successor", retire_sandbox_id="prior", create_kwargs={}, repository_image=False, + service_launch=VMServiceLaunch(False, False, False, 8080, 6080, 7680, []), ) diff --git a/packages/modal-infra/tests/test_snapshot_timeout.py b/packages/modal-infra/tests/test_snapshot_timeout.py index a4e4eb8ff6..a57d3fa0a5 100644 --- a/packages/modal-infra/tests/test_snapshot_timeout.py +++ b/packages/modal-infra/tests/test_snapshot_timeout.py @@ -21,7 +21,13 @@ async def test_pending_vm_reference_recovers_owned_allocation(monkeypatch): sandbox = SimpleNamespace( object_id="sb-owned", - get_tags=_async_method(docker_allocation_tags("session", "generation")), + get_tags=_async_method( + { + **docker_allocation_tags("session", "generation"), + "openinspect_vm_launch": "1-000-8080-6080-7680", + "openinspect_vm_ports": "none", + } + ), ) lookup = _async_method(sandbox) monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.from_name", lookup) @@ -34,7 +40,13 @@ async def test_pending_vm_reference_recovers_owned_allocation(monkeypatch): async def test_pending_vm_reference_stops_only_its_owned_allocation(monkeypatch): sandbox = SimpleNamespace( object_id="sb-owned", - get_tags=_async_method(docker_allocation_tags("session", "generation")), + get_tags=_async_method( + { + **docker_allocation_tags("session", "generation"), + "openinspect_vm_launch": "1-000-8080-6080-7680", + "openinspect_vm_ports": "none", + } + ), terminate=_async_method(), ) from_name = _async_method(sandbox) diff --git a/packages/modal-infra/tests/test_vm_resolve.py b/packages/modal-infra/tests/test_vm_resolve.py new file mode 100644 index 0000000000..a689d7fb28 --- /dev/null +++ b/packages/modal-infra/tests/test_vm_resolve.py @@ -0,0 +1,434 @@ +"""Generation-checked, lookup-only VM recovery API.""" + +import json +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest +from fastapi import HTTPException +from modal.exception import AlreadyExistsError, NotFoundError + +from sandbox_runtime.constants import ( + CODE_SERVER_PORT_ENV_VAR, + EXPECTED_TUNNEL_PORTS_ENV_VAR, + NOVNC_PORT_ENV_VAR, + TTYD_PROXY_PORT_ENV_VAR, + VNC_PASSWORD_ENV_VAR, +) +from src import web_api +from src.sandbox import manager as manager_module +from src.sandbox.launch_policy import docker_allocation_name, docker_allocation_tags +from src.sandbox.tunnels import SandboxTunnels, TunnelUrls + +SESSION = "session-1" +GENERATION = "generation-1" +RESOLVE_REQUEST = {"session_id": SESSION, "sandbox_id": GENERATION} + + +def _tags(launch="1-111-9000-9001-9002", ports="3000-3001"): + return { + **docker_allocation_tags(SESSION, GENERATION), + "openinspect_vm_launch": launch, + "openinspect_vm_ports": ports, + } + + +async def _call(endpoint, request, authorization="Bearer test"): + return await endpoint.get_raw_f()( + request, + authorization=authorization, + x_trace_id=None, + x_request_id=None, + x_session_id=None, + x_sandbox_id=None, + ) + + +def _sandbox(tags, env=None): + async def execute(*args, **kwargs): + keys = args[4:] + output = json.dumps({key: (env or {}).get(key) for key in keys}) + return SimpleNamespace( + stdout=SimpleNamespace(read=SimpleNamespace(aio=AsyncMock(return_value=output))), + wait=SimpleNamespace(aio=AsyncMock(return_value=0)), + ) + + return SimpleNamespace( + object_id="sb-real-id", + get_tags=SimpleNamespace(aio=AsyncMock(return_value=tags)), + exec=SimpleNamespace(aio=AsyncMock(side_effect=execute)), + terminate=SimpleNamespace(aio=AsyncMock()), + ) + + +@pytest.mark.asyncio +async def test_resolve_returns_owned_vm_id_access_and_tunnels_without_mutation(monkeypatch): + monkeypatch.setattr(web_api, "require_auth", lambda _token: None) + env = { + "CODE_SERVER_PASSWORD": "original-code-password", + VNC_PASSWORD_ENV_VAR: "original-vnc-password", + CODE_SERVER_PORT_ENV_VAR: "9000", + NOVNC_PORT_ENV_VAR: "9001", + TTYD_PROXY_PORT_ENV_VAR: "9002", + EXPECTED_TUNNEL_PORTS_ENV_VAR: "3000,3001", + "TERMINAL_ENABLED": "true", + } + sandbox = _sandbox(_tags(), env) + from_name = AsyncMock(return_value=sandbox) + create = AsyncMock(side_effect=AssertionError("resolve must not create")) + monkeypatch.setattr(manager_module.modal.Sandbox, "from_name", SimpleNamespace(aio=from_name)) + monkeypatch.setattr(manager_module.modal.Sandbox, "create", SimpleNamespace(aio=create)) + tunnels = AsyncMock( + return_value=TunnelUrls( + "https://code.example", + "https://vnc.example", + "https://terminal.example", + {3000: "https://app.example", 3001: "https://other.example"}, + ) + ) + monkeypatch.setattr(SandboxTunnels, "resolve", tunnels) + + result = await _call(web_api.api_resolve_vm_sandbox, RESOLVE_REQUEST) + + assert result == { + "success": True, + "data": { + "sandbox_id": GENERATION, + "modal_object_id": "sb-real-id", + "code_server_url": "https://code.example", + "code_server_password": "original-code-password", + "vnc_url": "https://vnc.example", + "vnc_password": "original-vnc-password", + "ttyd_url": "https://terminal.example", + "tunnel_urls": {3000: "https://app.example", 3001: "https://other.example"}, + "sandbox_backend": "modal-vm", + }, + } + from_name.assert_awaited_once_with("open-inspect", docker_allocation_name(SESSION)) + sandbox.get_tags.aio.assert_awaited_once_with() + sandbox.exec.aio.assert_awaited_once() + sandbox.terminate.aio.assert_not_awaited() + create.assert_not_awaited() + tunnels.assert_awaited_once_with(sandbox, GENERATION, write_env_file=False) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("missing_port", [9000, 9001, 9002, 3001]) +async def test_resolve_retries_when_enabled_tunnel_is_missing(monkeypatch, missing_port): + monkeypatch.setattr(web_api, "require_auth", lambda _token: None) + sandbox = _sandbox( + _tags(), + { + "CODE_SERVER_PASSWORD": "original-code-password", + VNC_PASSWORD_ENV_VAR: "original-vnc-password", + CODE_SERVER_PORT_ENV_VAR: "9000", + NOVNC_PORT_ENV_VAR: "9001", + TTYD_PROXY_PORT_ENV_VAR: "9002", + EXPECTED_TUNNEL_PORTS_ENV_VAR: "3000,3001", + "TERMINAL_ENABLED": "true", + }, + ) + monkeypatch.setattr( + manager_module.modal.Sandbox, + "from_name", + SimpleNamespace(aio=AsyncMock(return_value=sandbox)), + ) + create = AsyncMock(side_effect=AssertionError("resolve must not create")) + monkeypatch.setattr(manager_module.modal.Sandbox, "create", SimpleNamespace(aio=create)) + monkeypatch.setattr( + SandboxTunnels, + "_resolve_tunnels", + AsyncMock( + return_value={ + port: f"https://port-{port}.example" + for port in [9000, 9001, 9002, 3000, 3001] + if port != missing_port + } + ), + ) + write_env = AsyncMock(side_effect=AssertionError("resolve must not write")) + monkeypatch.setattr(SandboxTunnels, "_write_tunnel_env_file", write_env) + + with pytest.raises(HTTPException) as exc: + await _call(web_api.api_resolve_vm_sandbox, RESOLVE_REQUEST) + + assert (exc.value.status_code, exc.value.detail) == (409, "race_pending") + create.assert_not_awaited() + sandbox.terminate.aio.assert_not_awaited() + write_env.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_resolve_disabled_access_does_not_return_credentials(monkeypatch): + monkeypatch.setattr(web_api, "require_auth", lambda _token: None) + sandbox = _sandbox(_tags("1-000-8080-6080-7680", "none")) + monkeypatch.setattr( + manager_module.modal.Sandbox, + "from_name", + SimpleNamespace(aio=AsyncMock(return_value=sandbox)), + ) + tunnels = AsyncMock(return_value=TunnelUrls()) + monkeypatch.setattr(SandboxTunnels, "resolve", tunnels) + + result = await _call(web_api.api_resolve_vm_sandbox, RESOLVE_REQUEST) + + assert result["data"]["code_server_password"] is None + assert result["data"]["vnc_password"] is None + tunnels.assert_awaited_once_with(sandbox, GENERATION, write_env_file=False) + + +@pytest.mark.asyncio +async def test_resolve_extra_tunnels_does_not_write_into_vm(monkeypatch): + monkeypatch.setattr(web_api, "require_auth", lambda _token: None) + sandbox = _sandbox( + _tags("1-000-8080-6080-7680", "3000"), + {EXPECTED_TUNNEL_PORTS_ENV_VAR: "9999"}, + ) + monkeypatch.setattr( + manager_module.modal.Sandbox, + "from_name", + SimpleNamespace(aio=AsyncMock(return_value=sandbox)), + ) + monkeypatch.setattr( + SandboxTunnels, + "_resolve_tunnels", + AsyncMock(return_value={3000: "https://app.example"}), + ) + write_env = AsyncMock(side_effect=AssertionError("resolve must not write")) + monkeypatch.setattr(SandboxTunnels, "_write_tunnel_env_file", write_env) + + result = await _call(web_api.api_resolve_vm_sandbox, RESOLVE_REQUEST) + + assert result["data"]["tunnel_urls"] == {3000: "https://app.example"} + write_env.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_resolve_legacy_user_password_does_not_enable_access(monkeypatch): + monkeypatch.setattr(web_api, "require_auth", lambda _token: None) + sandbox = _sandbox( + docker_allocation_tags(SESSION, GENERATION), + { + "CODE_SERVER_PASSWORD": "user-repo-secret", + VNC_PASSWORD_ENV_VAR: "user-vnc-secret", + CODE_SERVER_PORT_ENV_VAR: "9000", + EXPECTED_TUNNEL_PORTS_ENV_VAR: "3000", + "TERMINAL_ENABLED": "true", + }, + ) + monkeypatch.setattr( + manager_module.modal.Sandbox, + "from_name", + SimpleNamespace(aio=AsyncMock(return_value=sandbox)), + ) + tunnels = AsyncMock(side_effect=AssertionError("legacy resolve must not inspect tunnels")) + monkeypatch.setattr(SandboxTunnels, "resolve", tunnels) + + result = await _call(web_api.api_resolve_vm_sandbox, RESOLVE_REQUEST) + + assert result["data"] == { + "sandbox_id": GENERATION, + "modal_object_id": "sb-real-id", + "code_server_url": None, + "code_server_password": None, + "vnc_url": None, + "vnc_password": None, + "ttyd_url": None, + "tunnel_urls": None, + "sandbox_backend": "modal-vm", + } + sandbox.exec.aio.assert_not_awaited() + sandbox.terminate.aio.assert_not_awaited() + tunnels.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_resolve_versioned_flags_ignore_user_password_when_service_disabled(monkeypatch): + monkeypatch.setattr(web_api, "require_auth", lambda _token: None) + sandbox = _sandbox( + _tags("1-010-9000-9001-9002", "none"), + {"CODE_SERVER_PASSWORD": "user-repo-secret", VNC_PASSWORD_ENV_VAR: "vnc-password"}, + ) + monkeypatch.setattr( + manager_module.modal.Sandbox, + "from_name", + SimpleNamespace(aio=AsyncMock(return_value=sandbox)), + ) + tunnels = AsyncMock(return_value=TunnelUrls(vnc_url="https://vnc.example")) + monkeypatch.setattr(SandboxTunnels, "resolve", tunnels) + + result = await _call(web_api.api_resolve_vm_sandbox, RESOLVE_REQUEST) + + assert result["data"]["code_server_password"] is None + assert result["data"]["code_server_url"] is None + assert result["data"]["vnc_password"] == "vnc-password" + assert sandbox.exec.aio.call_args.args[-1:] == (VNC_PASSWORD_ENV_VAR,) + tunnels.assert_awaited_once_with(sandbox, GENERATION, write_env_file=False) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "launch,ports", + [ + ("2-111-9000-9001-9002", "3000"), + ("1-111-9000-9001-9002", None), + ("1-111-0-9001-9002", "3000"), + ("1-11x-9000-9001-9002", "3000"), + ("1-111-9000-9001-9002", "65536"), + ], +) +async def test_resolve_unknown_or_incomplete_metadata_never_falls_back_to_env( + monkeypatch, launch, ports +): + monkeypatch.setattr(web_api, "require_auth", lambda _token: None) + tags = _tags(launch, ports) + if ports is None: + del tags["openinspect_vm_ports"] + sandbox = _sandbox(tags, {"CODE_SERVER_PASSWORD": "user-repo-secret"}) + monkeypatch.setattr( + manager_module.modal.Sandbox, + "from_name", + SimpleNamespace(aio=AsyncMock(return_value=sandbox)), + ) + tunnels = AsyncMock(side_effect=AssertionError("invalid metadata must not inspect tunnels")) + monkeypatch.setattr(SandboxTunnels, "resolve", tunnels) + + result = await _call(web_api.api_resolve_vm_sandbox, RESOLVE_REQUEST) + + assert result["data"]["modal_object_id"] == "sb-real-id" + assert result["data"]["code_server_password"] is None + assert result["data"]["tunnel_urls"] is None + sandbox.exec.aio.assert_not_awaited() + tunnels.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("allocation", "status", "detail"), + [ + (None, 409, "not_visible"), + ("foreign", 409, "other_generation"), + ], +) +async def test_resolve_reports_typed_absence_or_foreign_generation( + monkeypatch, allocation, status, detail +): + monkeypatch.setattr(web_api, "require_auth", lambda _token: None) + lookup = ( + AsyncMock(side_effect=NotFoundError("not visible")) + if allocation is None + else AsyncMock(return_value=_sandbox(docker_allocation_tags(SESSION, "other"))) + ) + monkeypatch.setattr(manager_module.modal.Sandbox, "from_name", SimpleNamespace(aio=lookup)) + create = AsyncMock() + monkeypatch.setattr(manager_module.modal.Sandbox, "create", SimpleNamespace(aio=create)) + + with pytest.raises(HTTPException) as exc: + await _call(web_api.api_resolve_vm_sandbox, RESOLVE_REQUEST) + + assert (exc.value.status_code, exc.value.detail) == (status, detail) + create.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_resolve_rejects_unexpected_allocation_tags(monkeypatch): + monkeypatch.setattr(web_api, "require_auth", lambda _token: None) + sandbox = _sandbox({**_tags(), "unexpected": "tag"}) + monkeypatch.setattr( + manager_module.modal.Sandbox, + "from_name", + SimpleNamespace(aio=AsyncMock(return_value=sandbox)), + ) + + with pytest.raises(HTTPException) as exc: + await _call(web_api.api_resolve_vm_sandbox, RESOLVE_REQUEST) + + assert (exc.value.status_code, exc.value.detail) == (409, "other_generation") + sandbox.exec.aio.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_resolve_authenticates_before_lookup_or_validation(monkeypatch): + lookup = AsyncMock() + monkeypatch.setattr(manager_module.modal.Sandbox, "from_name", SimpleNamespace(aio=lookup)) + monkeypatch.setattr( + web_api, + "require_auth", + lambda _token: (_ for _ in ()).throw(HTTPException(status_code=401)), + ) + + with pytest.raises(HTTPException) as exc: + await _call(web_api.api_resolve_vm_sandbox, {"sandbox_auth_token": "secret"}, None) + + assert exc.value.status_code == 401 + lookup.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "extra", ["sandbox_auth_token", "user_env_vars", "retire_sandbox_id", "control_plane_url"] +) +async def test_resolve_rejects_secret_or_mutating_request_fields(monkeypatch, extra): + monkeypatch.setattr(web_api, "require_auth", lambda _token: None) + lookup = AsyncMock() + monkeypatch.setattr(manager_module.modal.Sandbox, "from_name", SimpleNamespace(aio=lookup)) + + with pytest.raises(HTTPException) as exc: + await _call(web_api.api_resolve_vm_sandbox, {**RESOLVE_REQUEST, extra: "forbidden"}) + + assert exc.value.status_code == 400 + lookup.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("field", ["session_id", "sandbox_id"]) +async def test_resolve_requires_both_identity_fields(monkeypatch, field): + monkeypatch.setattr(web_api, "require_auth", lambda _token: None) + + with pytest.raises(HTTPException) as exc: + await _call( + web_api.api_resolve_vm_sandbox, {k: v for k, v in RESOLVE_REQUEST.items() if k != field} + ) + + assert exc.value.status_code == 400 + assert exc.value.detail == f"{field} is required" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("endpoint", ["api_create_sandbox", "api_restore_sandbox"]) +@pytest.mark.parametrize( + ("case", "detail"), + [("foreign", "other_generation"), ("expired", "window_closed"), ("race", "race_pending")], +) +async def test_vm_launch_reports_typed_outcomes(monkeypatch, endpoint, case, detail): + monkeypatch.setattr(web_api, "require_auth", lambda _token: None) + monkeypatch.setattr(web_api, "require_valid_control_plane_url", lambda _url: None) + monkeypatch.setattr("src.images.base.docker_image", object()) + monkeypatch.setattr(manager_module.modal.Image, "from_id", lambda _id: object()) + lookup = ( + AsyncMock(return_value=_sandbox(docker_allocation_tags(SESSION, "other"))) + if case == "foreign" + else AsyncMock(side_effect=NotFoundError("not visible")) + ) + create = AsyncMock(side_effect=AlreadyExistsError("winner not visible")) + monkeypatch.setattr(manager_module.modal.Sandbox, "from_name", SimpleNamespace(aio=lookup)) + monkeypatch.setattr(manager_module.modal.Sandbox, "create", SimpleNamespace(aio=create)) + request = { + "sandbox_id": GENERATION, + "control_plane_url": "https://control.example", + "sandbox_auth_token": "secret", + "sandbox_backend": "modal-vm", + "launch_deadline_at_ms": 1 if case == "expired" else 9999999999999, + } + if endpoint == "api_create_sandbox": + request["session_id"] = SESSION + else: + request["session_config"] = {"session_id": SESSION} + request["snapshot_image_id"] = "im-snapshot" + + with pytest.raises(HTTPException) as exc: + await _call(getattr(web_api, endpoint), request) + + assert (exc.value.status_code, exc.value.detail) == (409, detail) + assert create.await_count == (1 if case == "race" else 0) + assert lookup.await_count == (2 if case == "race" else 1) From 0530683b1e3567c69014498e4c52593f76d0c187 Mon Sep 17 00:00:00 2001 From: Cole Murray Date: Mon, 28 Sep 2026 20:33:59 -0700 Subject: [PATCH 04/13] feat(control-plane): enforce session routes behind teams flag (#2118) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - Gate all 35 active-user session item routes on the persisted session row, with private access enforced in every mode and `TEAMS_ENFORCEMENT` defaulting to `shadow`. - Centralize per-session admission for item and batch routes. The existing actorless route grant authorizes the action after the service viewer passes visibility; private sessions remain concealed. Child summary and cancel routes check both the parent and child rows. - Emit one response-time shadow audit decision with the actual HTTP status, including batch target observations. Record every permitted private Owner break-glass read before later checks; classify and label it as an audit operation. - Preserve legacy bulk-only custom-role access in `off` and `shadow`; keep the private snapshot sandbox redaction in all modes. Cache mode and memberships per request, validate the Node flag at startup, and return 503 for invalid Worker flag values. Issue: https://linear.app/colemurray/issue/COL-198/teams-pr-5-control-plane-session-route-requirement-on-every-sessionsid ## Checkpoint **Validation (latest commit):** - `npm run build -w @open-inspect/shared`: passed. - `npm run typecheck`: passed across all workspaces. - `npm run lint:fix`, `npm run format:check`: passed. - `npm run lint:sql-portability`: clean (24 baselined occurrences across 4 files). - `npm run lint:complexity`: report-only, command passed. - `npm test -w @open-inspect/control-plane`: 330 files, 5,301 tests passed. - `npm run test:integration -w @open-inspect/control-plane`: 121 files, 1,455 tests passed, 1 skipped. Existing forced-eviction and NDJSON diagnostics remain but the command exits successfully. - `npm test -w @open-inspect/web`: 226 files, 1,977 tests passed. - Targeted shared audit and session-access tests: 57 passed. **Red checkpoints fixed during implementation (verbatim excerpts):** ``` AssertionError: GET /sessions/:id/children/:childId: expected false to be true // Object.is equality AssertionError: expected 500 to be 503 // Object.is equality AssertionError: expected [ { …(2) }, { …(2) } ] to have a length of 1 but got 2 ``` The first full integration run also found an older RBAC fixture that lacked a parent row despite asserting a create-permission denial. Session-first admission correctly returned 404; the test now seeds its parent: ``` FAIL test/integration/rbac-routes.test.ts > RBAC routes > requires sessions.create in addition to parent collaboration when spawning a child AssertionError: expected 404 to be 403 // Object.is equality ``` The frozen catalog snapshot was refreshed for the child requirement declarations. All suites subsequently passed. **Main drift:** `SessionIndexStore.get` returns a validated `SessionEntry` rather than `SessionRow` (`session-index.ts:393-397`); admission maps it to the shared access row. The shared resolver is exported at the package root rather than through a `types/session-access` subpath. Single-export registration moved to `session-export.ts:351-368` on the current main; the route inventory still contains 35 active-user and 10 `none` session item routes. **Deliberately excluded:** No D1 or DO migration. Bulk `/sessions/export` scoping belongs to PR6, WebSocket subscribe/command checks to PR7, and child-list filtering plus visibility/collaborator creation endpoints to PR8. Batch `skipped` is a wire change to a strict shared response schema; no consumer parses that response body today. ## Summary by CodeRabbit * **New Features** * Added configurable team access enforcement with `off`, `shadow` (the default), and `on` modes. Private sessions remain restricted in every mode, with authorized break-glass access recorded in the audit log. * With enforcement enabled, team membership, role, and session collaboration determine access. Deleting a session requires its owner, a team lead, or a workspace administrator. * Shadow mode records access decisions that enforcement would deny while preserving legacy access behavior. * Batch archive responses now list sessions skipped because they were not found or the requester lacked permission. --------- Co-authored-by: Cole Murray <2492022+ColeMurray@users.noreply.github.com> Co-authored-by: waclaude --- .env.example | 3 + CHANGELOG.md | 8 + .../src/authorization/request-audit.ts | 64 ++- .../src/authorization/session-admission.ts | 107 ++++ .../authorization/teams-enforcement.test.ts | 26 + .../src/authorization/teams-enforcement.ts | 28 ++ .../src/db/session-collaborators.ts | 43 ++ .../control-plane/src/http/request-context.ts | 10 + .../control-plane/src/node/config.test.ts | 6 + packages/control-plane/src/node/config.ts | 3 + .../control-plane/src/router.policy.test.ts | 46 +- .../src/router.scm-credentials.test.ts | 8 + .../src/router.session-prompt.test.ts | 19 +- .../src/router.spawn-child.test.ts | 6 +- .../control-plane/src/router.test-support.ts | 47 ++ .../src/routes/session-attachments.ts | 6 +- .../src/routes/session-batch-archive.ts | 28 +- .../src/routes/session-child-spawn.ts | 5 +- .../src/routes/session-children.ts | 16 +- .../control-plane/src/routes/session-diffs.ts | 8 +- .../src/routes/session-export.test.ts | 9 +- .../src/routes/session-export.ts | 19 +- .../control-plane/src/routes/session-index.ts | 5 +- .../src/routes/session-media-stream.ts | 4 +- .../src/routes/session-media-upload.ts | 4 +- .../src/routes/session-prompt.ts | 4 +- .../src/routes/session-pull-requests.ts | 4 +- .../control-plane/src/routes/session-route.ts | 2 +- .../src/routes/session-runtime-proxy.test.ts | 118 +++-- .../src/routes/session-runtime-proxy.ts | 36 +- .../src/routes/session-skills.ts | 4 +- .../src/routes/session-ws-token.ts | 5 +- packages/control-plane/src/routes/shared.ts | 24 +- .../control-plane/src/routes/slack-notify.ts | 9 +- .../control-plane/src/routing/hono-app.ts | 9 +- .../src/routing/route-admission.ts | 84 +++- packages/control-plane/src/types.ts | 1 + ...ono-route-catalog-conformance.test.ts.snap | 70 +-- .../route-admission-matrix.test.ts.snap | 98 ++++ .../control-plane/test/integration/helpers.ts | 41 +- .../test/integration/rbac-routes.test.ts | 7 + .../route-admission-matrix.test.ts | 157 +++++- .../integration/session-access-routes.test.ts | 473 ++++++++++++++++++ .../integration/session-batch-archive.test.ts | 10 +- .../shared/src/types/audit-events.test.ts | 1 + packages/shared/src/types/audit-events.ts | 1 + packages/shared/src/types/session-archive.ts | 6 + .../settings/audit-log-settings.test.tsx | 6 + .../settings/audit-log-settings.tsx | 1 + terraform/environments/aws-production/main.tf | 1 + .../environments/production/variables.tf | 11 + .../production/workers-control-plane.tf | 1 + 52 files changed, 1526 insertions(+), 186 deletions(-) create mode 100644 packages/control-plane/src/authorization/session-admission.ts create mode 100644 packages/control-plane/src/authorization/teams-enforcement.test.ts create mode 100644 packages/control-plane/src/authorization/teams-enforcement.ts create mode 100644 packages/control-plane/src/db/session-collaborators.ts create mode 100644 packages/control-plane/test/integration/session-access-routes.test.ts diff --git a/.env.example b/.env.example index 90157511eb..ce5071b47a 100644 --- a/.env.example +++ b/.env.example @@ -256,6 +256,9 @@ EXECUTION_TIMEOUT_MS= # "enforce" (default) fails spawn and build on oversized secret payloads; # "warn" only logs. Cloudflare: not set by Terraform. SECRETS_CAP_ENFORCEMENT= +# Session team enforcement: off, shadow, or on. Unset defaults to shadow. +# Private visibility applies in every mode. Cloudflare: var.teams_enforcement. +TEAMS_ENFORCEMENT= # --------------------------------------------------------------------------- # Logging diff --git a/CHANGELOG.md b/CHANGELOG.md index 76d0d7a845..b0ae09f966 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,14 @@ New features, integrations, and notable improvements to Open-Inspect — newest first. +## September 29, 2026 + +### Added + +`TEAMS_ENFORCEMENT` controls session route access (`off`, `shadow` by default, or `on`). Private +sessions remain restricted in every mode; with enforcement on, session deletion requires the owner, +a team lead, or a workspace administrator. + ## September 28, 2026 ### Added diff --git a/packages/control-plane/src/authorization/request-audit.ts b/packages/control-plane/src/authorization/request-audit.ts index 1deb537ced..70226d559f 100644 --- a/packages/control-plane/src/authorization/request-audit.ts +++ b/packages/control-plane/src/authorization/request-audit.ts @@ -28,6 +28,8 @@ export type RouteAuthorizationDecision = kind: "allowed"; admission: "user" | "service" | "sandbox"; auditAllowed: boolean; + shadowReason?: string; + shadowDenials?: readonly { sessionId: string; reason: string }[]; }) | (AuthorizationDecisionEvidence & { kind: "denied"; @@ -39,7 +41,7 @@ export type RouteAuthorizationDecision = export function shouldAuditAllowedDecision( decision: Extract ): boolean { - return decision.auditAllowed; + return decision.auditAllowed || !!decision.shadowReason || !!decision.shadowDenials?.length; } /** @@ -58,6 +60,7 @@ export async function auditRouteAuthorizationDecision(input: { path: string; response: Response; decision: RouteAuthorizationDecision; + teamId?: string | null; }): Promise { const principal = input.ctx.principal; if (!principal) return; @@ -75,6 +78,20 @@ export async function auditRouteAuthorizationDecision(input: { const action = allowed ? AUTHORIZATION_DECISION_ACTIONS.allowed : AUTHORIZATION_DECISION_ACTIONS.denied; + const shadowCode = + decision.kind === "allowed" + ? decision.shadowDenials?.length + ? "shadow_denied:batch" + : decision.shadowReason + ? `shadow_denied:${decision.shadowReason}` + : null + : null; + const teamId = + input.teamId !== undefined + ? input.teamId + : input.ctx.childSessionAdmission + ? input.ctx.childSessionAdmission.row.ownerTeamId + : (input.ctx.sessionAdmission?.row.ownerTeamId ?? null); const metadata = { schema: AUTHORIZATION_DECISION_METADATA_SCHEMA, httpMethod: input.method, @@ -85,11 +102,14 @@ export async function auditRouteAuthorizationDecision(input: { ? { effectivePermissions: decision.effectivePermissions } : {}), ...(requiredPermission ? { requiredPermission } : {}), - responseCode: decision.kind === "denied" ? decision.reasonCode : null, + responseCode: decision.kind === "denied" ? decision.reasonCode : shadowCode, responseReason: decision.kind === "denied" ? decision.reason : null, requestId: input.ctx.request_id, traceId: input.ctx.trace_id, ...(decision.kind === "allowed" ? { admission: decision.admission } : {}), + ...(decision.kind === "allowed" && decision.shadowDenials?.length + ? { shadowDenials: decision.shadowDenials } + : {}), ...(principal.kind === "service" && principal.actor ? { actor: { @@ -108,8 +128,8 @@ export async function auditRouteAuthorizationDecision(input: { `INSERT INTO authorization_audit_events (id, occurred_at, request_id, principal_kind, actor_user_id_snapshot, actor_service_snapshot, action, resource_type, resource_id, - reason_code, operation_result, metadata_json) - VALUES (?, ?, ?, ?, ?, ?, ?, 'http_route', ?, ?, ?, ?)` + reason_code, operation_result, metadata_json, team_id) + VALUES (?, ?, ?, ?, ?, ?, ?, 'http_route', ?, ?, ?, ?, ?)` ) .bind( crypto.randomUUID(), @@ -120,9 +140,10 @@ export async function auditRouteAuthorizationDecision(input: { principal.kind === "service" ? principal.service : null, action, input.path, - decision.kind === "allowed" ? "authorization_allowed" : decision.reasonCode, + decision.kind === "allowed" ? (shadowCode ?? "authorization_allowed") : decision.reasonCode, allowed ? "applied" : "denied", - JSON.stringify(metadata) + JSON.stringify(metadata), + teamId ) .run(); } catch (cause) { @@ -135,3 +156,34 @@ export async function auditRouteAuthorizationDecision(input: { }); } } + +export async function auditPrivateSessionBreakGlass( + ctx: RequestContext, + sessionId: string, + teamId: string | null +): Promise { + const principal = ctx.principal; + const actorUserId = ctx.authorization?.userId; + if (!principal || !actorUserId) throw new Error("Missing private session break-glass actor"); + await ctx.db + .prepare( + `INSERT INTO authorization_audit_events + (id, occurred_at, request_id, principal_kind, actor_user_id_snapshot, + actor_service_snapshot, action, resource_type, resource_id, team_id, + reason_code, operation_result, metadata_json) + VALUES (?, ?, ?, ?, ?, ?, 'session.private_break_glass', 'session', ?, ?, ?, 'applied', ?)` + ) + .bind( + crypto.randomUUID(), + Date.now(), + ctx.request_id, + principal.kind, + actorUserId, + principal.kind === "service" ? principal.service : null, + sessionId, + teamId, + "session.private_break_glass", + JSON.stringify({ before: {}, requested: {}, after: {} }) + ) + .run(); +} diff --git a/packages/control-plane/src/authorization/session-admission.ts b/packages/control-plane/src/authorization/session-admission.ts new file mode 100644 index 0000000000..022c20f4ff --- /dev/null +++ b/packages/control-plane/src/authorization/session-admission.ts @@ -0,0 +1,107 @@ +import { checkSessionAccess, type SessionAction, type SessionViewer } from "@open-inspect/shared"; +import type { PermissionId } from "@open-inspect/shared/rbac"; +import type { TeamRole } from "@open-inspect/shared/types/teams"; +import { SessionCollaboratorStore } from "../db/session-collaborators"; +import { SessionIndexStore } from "../db/session-index"; +import { TeamMembershipStore } from "../db/team-memberships"; +import type { RequestContext } from "../http/request-context"; +import type { Env } from "../types"; +import { auditPrivateSessionBreakGlass } from "./request-audit"; +import { + legacyPermissionForAction, + parseTeamsEnforcementMode, + type TeamsEnforcementMode, +} from "./teams-enforcement"; + +export function teamsEnforcementMode(ctx: RequestContext, env: Env): TeamsEnforcementMode { + return (ctx.teamsEnforcementMode ??= parseTeamsEnforcementMode(env.TEAMS_ENFORCEMENT)); +} + +export function viewerFromContext( + ctx: RequestContext, + memberships: ReadonlyMap +): SessionViewer { + const authorization = ctx.authorization; + if (!authorization) { + if (ctx.principal?.kind === "service" && !ctx.principal.actor) + return { kind: "service", teamId: null }; + throw new Error("Missing request authorization"); + } + return { + kind: "user", + userId: authorization.userId, + roleKey: authorization.role.key, + permissions: authorization.permissions, + suspended: authorization.suspendedAt !== null, + memberships, + }; +} + +export type SessionAdmissionOutcome = + | { kind: "not_found" } + | { kind: "action_denied"; reason: string } + | { kind: "allowed"; legacyPermission: PermissionId | null }; + +/** Resolve one D1 session; a null slot is used by body-ID batches, not item routes. */ +export async function evaluateSessionAdmission( + ctx: RequestContext, + env: Env, + sessionId: string, + action: SessionAction, + slot: "session" | "child" | null = "session" +): Promise { + const mode = teamsEnforcementMode(ctx, env); + const row = await new SessionIndexStore(ctx.db).get(sessionId); + if (!row) return { kind: "not_found" }; + + if (mode === "off" && row.visibility !== "private") { + return { kind: "allowed", legacyPermission: legacyPermissionForAction(action) }; + } + + const memberships = + mode === "off" || !ctx.authorization + ? new Map() + : (ctx.sessionMemberships ??= await new TeamMembershipStore(ctx.db).listForUser( + ctx.authorization.userId + )); + const viewer = viewerFromContext(ctx, memberships); + const accessRow = { + ...row, + ownerUserId: row.userId ?? null, + collaboratorIds: await new SessionCollaboratorStore(ctx.db).listUserIds(sessionId), + }; + if (slot === "session") ctx.sessionAdmission = { row: accessRow, viewer }; + if (slot === "child") ctx.childSessionAdmission = { row: accessRow, viewer }; + + const read = checkSessionAccess(viewer, accessRow, "read"); + if ( + !read.allowed && + (mode === "on" || (row.visibility === "private" && read.reason === "private")) + ) { + return { kind: "not_found" }; + } + if (read.allowed && read.audit === "session.private_break_glass") { + await auditPrivateSessionBreakGlass(ctx, sessionId, row.ownerTeamId); + } + + // The signed route grant authorizes actorless actions; the service resolver only checks visibility. + const decision = viewer.kind === "service" ? null : checkSessionAccess(viewer, accessRow, action); + if (mode === "on" && decision && !decision.allowed) { + return { kind: "action_denied", reason: decision.reason }; + } + if (mode === "shadow") { + const reason = !read.allowed + ? read.reason + : decision && !decision.allowed + ? decision.reason + : null; + if (reason) { + if (slot === null) (ctx.shadowBatchDenials ??= []).push({ sessionId, reason }); + else ctx.shadowSessionDenial ??= reason; + } + } + return { + kind: "allowed", + legacyPermission: mode === "on" ? null : legacyPermissionForAction(action), + }; +} diff --git a/packages/control-plane/src/authorization/teams-enforcement.test.ts b/packages/control-plane/src/authorization/teams-enforcement.test.ts new file mode 100644 index 0000000000..a9fd84ed59 --- /dev/null +++ b/packages/control-plane/src/authorization/teams-enforcement.test.ts @@ -0,0 +1,26 @@ +import { describe, expect, it } from "vitest"; +import { SESSION_ACTIONS } from "@open-inspect/shared"; +import { legacyPermissionForAction, parseTeamsEnforcementMode } from "./teams-enforcement"; + +describe("teams enforcement", () => { + it("defaults to shadow and accepts only the three modes", () => { + expect(parseTeamsEnforcementMode(undefined)).toBe("shadow"); + expect(parseTeamsEnforcementMode("off")).toBe("off"); + expect(parseTeamsEnforcementMode("shadow")).toBe("shadow"); + expect(parseTeamsEnforcementMode("on")).toBe("on"); + expect(() => parseTeamsEnforcementMode("enabled")).toThrow(); + }); + + it("maps every resolver action to its pre-enforcement permission", () => { + expect(SESSION_ACTIONS.map((action) => legacyPermissionForAction(action))).toEqual([ + "sessions.read", + "sessions.collaborate", + "sessions.lifecycle", + "sessions.delete", + "sessions.sandbox_access", + "sessions.lifecycle", + "sessions.lifecycle", + "sessions.lifecycle", + ]); + }); +}); diff --git a/packages/control-plane/src/authorization/teams-enforcement.ts b/packages/control-plane/src/authorization/teams-enforcement.ts new file mode 100644 index 0000000000..6edb6da94b --- /dev/null +++ b/packages/control-plane/src/authorization/teams-enforcement.ts @@ -0,0 +1,28 @@ +import type { PermissionId } from "@open-inspect/shared/rbac"; +import type { SessionAction } from "@open-inspect/shared"; + +export type TeamsEnforcementMode = "off" | "shadow" | "on"; + +export function parseTeamsEnforcementMode(value: string | undefined): TeamsEnforcementMode { + if (value === undefined || value === "") return "shadow"; + if (value === "off" || value === "shadow" || value === "on") return value; + throw new Error(`Invalid TEAMS_ENFORCEMENT: ${value}`); +} + +export function legacyPermissionForAction(action: SessionAction): PermissionId { + switch (action) { + case "read": + return "sessions.read"; + case "collaborate": + return "sessions.collaborate"; + case "delete": + return "sessions.delete"; + case "sandbox": + return "sessions.sandbox_access"; + case "lifecycle": + case "move": + case "changeVisibility": + case "manageCollaborators": + return "sessions.lifecycle"; + } +} diff --git a/packages/control-plane/src/db/session-collaborators.ts b/packages/control-plane/src/db/session-collaborators.ts new file mode 100644 index 0000000000..eb24384886 --- /dev/null +++ b/packages/control-plane/src/db/session-collaborators.ts @@ -0,0 +1,43 @@ +import { z } from "zod"; +import type { SqlDatabase } from "./sql-database"; + +const collaboratorSchema = z.object({ session_id: z.string(), user_id: z.string() }); + +export class SessionCollaboratorStore { + constructor(private readonly db: SqlDatabase) {} + + async listUserIds(sessionId: string): Promise { + const rows = await this.db + .prepare("SELECT session_id, user_id FROM session_collaborators WHERE session_id = ?") + .bind(sessionId) + .all(); + return rows.results.map((row) => collaboratorSchema.parse(row).user_id); + } + + async listForUser(userId: string): Promise { + const rows = await this.db + .prepare("SELECT session_id, user_id FROM session_collaborators WHERE user_id = ?") + .bind(userId) + .all(); + return rows.results.map((row) => collaboratorSchema.parse(row).session_id); + } + + async add(sessionId: string, userId: string, addedBy: string): Promise { + const result = await this.db + .prepare( + `INSERT INTO session_collaborators (session_id, user_id, added_by, created_at) + VALUES (?, ?, ?, ?) ON CONFLICT (session_id, user_id) DO NOTHING` + ) + .bind(sessionId, userId, addedBy, Date.now()) + .run(); + return result.meta.changes > 0; + } + + async remove(sessionId: string, userId: string): Promise { + const result = await this.db + .prepare("DELETE FROM session_collaborators WHERE session_id = ? AND user_id = ?") + .bind(sessionId, userId) + .run(); + return result.meta.changes > 0; + } +} diff --git a/packages/control-plane/src/http/request-context.ts b/packages/control-plane/src/http/request-context.ts index 4fc1c03cc8..8218c7bb62 100644 --- a/packages/control-plane/src/http/request-context.ts +++ b/packages/control-plane/src/http/request-context.ts @@ -1,12 +1,16 @@ import type { EffectiveAuthorization } from "@open-inspect/shared/rbac"; import type { TeamCapabilities } from "@open-inspect/shared/types/team-access"; import type { Team } from "@open-inspect/shared/types/teams"; +import type { TeamRole } from "@open-inspect/shared/types/teams"; +import type { SessionAccessRow, SessionViewer } from "@open-inspect/shared"; import type { AuthenticationContext, Principal } from "../auth/principal"; import type { AuthenticationRequestServices } from "../auth/request-services"; import type { UserAuthRuntime } from "../auth/user/runtime"; import type { AutomationRow } from "../db/automation-store"; +import type { SessionEntry } from "../db/session-index"; import type { RequestMetrics } from "../db/instrumented-sql-database"; import type { BackgroundTasks } from "../platform-ports"; +import type { TeamsEnforcementMode } from "../authorization/teams-enforcement"; /** Automation resource admitted for the current mutation. */ export interface AutomationRouteAdmission { @@ -27,4 +31,10 @@ export type RequestContext = AuthenticationRequestServices & { authorization?: EffectiveAuthorization; automationAdmission?: AutomationRouteAdmission; teamAdmission?: { team: Team; access: TeamCapabilities }; + sessionAdmission?: { row: SessionEntry & SessionAccessRow; viewer: SessionViewer }; + childSessionAdmission?: { row: SessionEntry & SessionAccessRow; viewer: SessionViewer }; + sessionMemberships?: ReadonlyMap; + teamsEnforcementMode?: TeamsEnforcementMode; + shadowSessionDenial?: string; + shadowBatchDenials?: { sessionId: string; reason: string }[]; }; diff --git a/packages/control-plane/src/node/config.test.ts b/packages/control-plane/src/node/config.test.ts index ab099d8291..1247b97482 100644 --- a/packages/control-plane/src/node/config.test.ts +++ b/packages/control-plane/src/node/config.test.ts @@ -26,6 +26,12 @@ describe("readEnvConfig", () => { expect("LOG_LEVEL" in config).toBe(false); }); + it("rejects invalid teams enforcement modes at startup", () => { + expect(() => readEnvConfig({ ...REQUIRED, TEAMS_ENFORCEMENT: "enabled" })).toThrow( + "Invalid TEAMS_ENFORCEMENT" + ); + }); + it("names every missing required variable at once", () => { expect(() => readEnvConfig({ DEPLOYMENT_NAME: "test", TOKEN_ENCRYPTION_KEY: "" })).toThrow( "Missing required configuration: GITHUB_BOT_USERNAME, TOKEN_ENCRYPTION_KEY, PROVIDER_ACCOUNTS_ENCRYPTION_KEY, REPO_SECRETS_ENCRYPTION_KEY" diff --git a/packages/control-plane/src/node/config.ts b/packages/control-plane/src/node/config.ts index 6c6f85a607..2db85230d5 100644 --- a/packages/control-plane/src/node/config.ts +++ b/packages/control-plane/src/node/config.ts @@ -17,6 +17,7 @@ import { dirname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import type { EnvConfig } from "../types"; +import { parseTeamsEnforcementMode } from "../authorization/teams-enforcement"; /** A source of configuration values, `process.env` in production. */ export type ConfigSource = Record; @@ -91,6 +92,7 @@ const ENV_CONFIG_KEYS = { SANDBOX_BOOT_TIMEOUT_MS: true, EXECUTION_TIMEOUT_MS: true, SECRETS_CAP_ENFORCEMENT: true, + TEAMS_ENFORCEMENT: true, LOG_LEVEL: true, } as const satisfies Record; @@ -155,6 +157,7 @@ export function readEnvConfig(source: ConfigSource): EnvConfig { if (missing.length > 0) { throw new Error(`Missing required configuration: ${missing.join(", ")}`); } + parseTeamsEnforcementMode(config.TEAMS_ENFORCEMENT); return config as EnvConfig; } diff --git a/packages/control-plane/src/router.policy.test.ts b/packages/control-plane/src/router.policy.test.ts index acc028981e..57dd559b1f 100644 --- a/packages/control-plane/src/router.policy.test.ts +++ b/packages/control-plane/src/router.policy.test.ts @@ -31,10 +31,10 @@ describe("route policy table", () => { }); }); - it("gates a single-session export with sessions.export", () => { + it("gates a single-session export with session read and sessions.export", () => { expect(routeFor("GET", "/sessions/session-1/export")?.authorization).toMatchObject({ kind: "active-user", - allOf: [{ permission: "sessions.export" }], + allOf: [{ kind: "session", action: "read" }, { permission: "sessions.export" }], }); }); @@ -117,6 +117,30 @@ describe("route policy table", () => { } }); + it("requires session admission on every active-user session item route", () => { + const exceptions: string[] = []; + for (const route of routes) { + if (!route.path.startsWith("/sessions/:id") || route.authorization.kind !== "active-user") + continue; + const identity = `${route.method} ${route.path}`; + if (exceptions.includes(identity)) continue; + expect( + route.authorization.allOf.some((requirement) => requirement.kind === "session"), + identity + ).toBe(true); + if (route.path.includes(":childId")) { + expect( + route.authorization.allOf.some( + (requirement) => + requirement.kind === "session" && requirement.sessionIdParam === "childId" + ), + identity + ).toBe(true); + } + } + expect(exceptions).toEqual([]); + }); + it.each([ ["GET", "/repos", [{ service: "slack-bot" }, { service: "linear-bot" }]], ["GET", "/repos/acme/widgets/metadata", [{ service: "github-bot" }]], @@ -233,7 +257,7 @@ describe("route policy table", () => { }); expect(routeFor("POST", "/sessions/session-1/ws-token")?.authorization).toMatchObject({ kind: "active-user", - allOf: [{ kind: "permission", permission: "sessions.read" }], + allOf: [{ kind: "session", action: "read" }], }); expect(routeFor("POST", "/sessions/session-1/stop")?.authorization).toMatchObject({ service: { kind: "actor", actorlessGrants: [{ service: "linear-bot" }] }, @@ -245,14 +269,26 @@ describe("route policy table", () => { expect(routeFor("POST", "/sessions/parent/children")?.authorization).toMatchObject({ kind: "active-user", allOf: [ + { kind: "session", action: "collaborate" }, { kind: "permission", permission: "sessions.create" }, - { kind: "permission", permission: "sessions.collaborate" }, ], }); expect(routeFor("GET", "/sessions/parent/children/child")?.authorization).toMatchObject({ kind: "active-user", - allOf: [{ kind: "permission", permission: "sessions.read" }], + allOf: [ + { kind: "session", action: "read", sessionIdParam: "id" }, + { kind: "session", action: "read", sessionIdParam: "childId" }, + ], }); + expect(routeFor("POST", "/sessions/parent/children/child/cancel")?.authorization).toMatchObject( + { + kind: "active-user", + allOf: [ + { kind: "session", action: "read", sessionIdParam: "id" }, + { kind: "session", action: "lifecycle", sessionIdParam: "childId" }, + ], + } + ); expect(routeFor("POST", "/internal/github-event")?.authorization).toMatchObject({ kind: "service", services: ["github-bot"], diff --git a/packages/control-plane/src/router.scm-credentials.test.ts b/packages/control-plane/src/router.scm-credentials.test.ts index 6e211fe616..974cb11591 100644 --- a/packages/control-plane/src/router.scm-credentials.test.ts +++ b/packages/control-plane/src/router.scm-credentials.test.ts @@ -7,6 +7,7 @@ import { signedServiceRequest, TEST_BACKGROUND_TASK_CONTEXT, TEST_SERVICE_SECRETS, + TEST_SESSION_ROW, } from "./router.test-support"; function routeFor(method: string, path: string) { @@ -55,6 +56,13 @@ function createEnv(options?: { actorAuthorized?: boolean }) { }; return identityStatement; } + if (options?.actorAuthorized && sql.includes("SELECT * FROM sessions")) { + const sessionStatement = { + bind: vi.fn(() => sessionStatement), + first: vi.fn(async () => TEST_SESSION_ROW), + }; + return sessionStatement; + } if ( options?.actorAuthorized && sql.includes("FROM users u") && diff --git a/packages/control-plane/src/router.session-prompt.test.ts b/packages/control-plane/src/router.session-prompt.test.ts index 90781a62dd..b015aed3b1 100644 --- a/packages/control-plane/src/router.session-prompt.test.ts +++ b/packages/control-plane/src/router.session-prompt.test.ts @@ -9,6 +9,7 @@ import { signedServiceRequest, TEST_BACKGROUND_TASK_CONTEXT, TEST_SERVICE_SECRETS, + TEST_SESSION_ROW, } from "./router.test-support"; vi.mock("./db/user-store", () => ({ @@ -73,11 +74,27 @@ function createEnv(sessionFetch: (request: Request) => Promise): Recor })), run: vi.fn(async () => ({ meta: { changes: 0 } })), }; + const sessionStatement = { + ...statement, + bind: vi.fn(() => sessionStatement), + first: vi.fn(async () => TEST_SESSION_ROW), + }; + const membershipStatement = { + ...statement, + bind: vi.fn(() => membershipStatement), + all: vi.fn(async () => ({ results: [] })), + }; return { ...TEST_SERVICE_SECRETS, SCM_PROVIDER: "github", DB: { - prepare: vi.fn(() => statement), + prepare: vi.fn((sql: string) => + sql.includes("SELECT * FROM sessions") + ? sessionStatement + : sql.includes("FROM team_memberships") || sql.includes("FROM session_collaborators") + ? membershipStatement + : statement + ), batch: vi.fn(), exec: vi.fn(), dump: vi.fn(), diff --git a/packages/control-plane/src/router.spawn-child.test.ts b/packages/control-plane/src/router.spawn-child.test.ts index 7496c6b84f..d80e08680b 100644 --- a/packages/control-plane/src/router.spawn-child.test.ts +++ b/packages/control-plane/src/router.spawn-child.test.ts @@ -660,7 +660,8 @@ describe("handleSpawnChild prompt enqueue handling", () => { expect(response.status).toBe(400); await expect(response.json()).resolves.toEqual({ error: "title and prompt are required" }); - expect(SessionIndexStore).not.toHaveBeenCalled(); + expect(store.get).toHaveBeenCalledWith(parentId); + expect(store.create).not.toHaveBeenCalled(); }); it("returns 400 for a child spawn body that is not JSON", async () => { @@ -689,7 +690,8 @@ describe("handleSpawnChild prompt enqueue handling", () => { expect(response.status).toBe(400); await expect(response.json()).resolves.toEqual({ error: "Invalid JSON body" }); - expect(SessionIndexStore).not.toHaveBeenCalled(); + expect(store.get).toHaveBeenCalledWith(parentId); + expect(store.create).not.toHaveBeenCalled(); }); it("returns 500 for a malformed parent spawn context", async () => { diff --git a/packages/control-plane/src/router.test-support.ts b/packages/control-plane/src/router.test-support.ts index 430680f35a..02fcc3cd92 100644 --- a/packages/control-plane/src/router.test-support.ts +++ b/packages/control-plane/src/router.test-support.ts @@ -12,6 +12,7 @@ import { createTestBackgroundTasks } from "./background-tasks.test-support"; import { BUILT_IN_ROLE_REGISTRY, type PermissionId } from "@open-inspect/shared/rbac"; import type { CacheStore } from "@open-inspect/shared/cache-store"; import type { SqlDatabase, SqlStatement } from "./db/sql-database"; +import type { SessionRow } from "./db/session-row"; import type { SessionRuntimeDispatch } from "./session/runtime-client"; import { cloudflareHost } from "./cloudflare/http-host"; import { createControlPlaneApp, type RouteModule } from "./routing/hono-app"; @@ -92,6 +93,52 @@ export function emptyStatement(): SqlStatement { return statement; } +/** A persisted workspace session for route tests that are about handler behavior. */ +export const TEST_SESSION_ROW: SessionRow = { + id: "session-1", + title: null, + repo_owner: null, + repo_name: null, + harness: "opencode", + model: "test/model", + reasoning_effort: null, + base_branch: null, + status: "completed", + parent_session_id: null, + root_session_id: null, + spawn_source: "user", + spawn_depth: 0, + automation_id: null, + automation_run_id: null, + scm_login: null, + user_id: TEST_USER_ID, + owner_team_id: null, + visibility: "workspace", + project_id: null, + total_cost: 0, + active_duration_ms: 0, + message_count: 0, + pr_count: 0, + input_tokens: 0, + output_tokens: 0, + reasoning_tokens: 0, + cache_read_tokens: 0, + cache_write_tokens: 0, + environment_id: null, + created_at: 1, + updated_at: 1, +}; + +export function sessionAdmissionStatement(sql: string): SqlStatement | null { + if (!sql.includes("SELECT * FROM sessions")) return null; + const statement: SqlStatement = { + ...emptyStatement(), + bind: () => statement, + first: async () => TEST_SESSION_ROW as T, + }; + return statement; +} + /** * A database whose effective-authorization lookup answers for one active * user, for request-level unit tests of admitted handlers. The two diff --git a/packages/control-plane/src/routes/session-attachments.ts b/packages/control-plane/src/routes/session-attachments.ts index 6032bf282f..76f08effde 100644 --- a/packages/control-plane/src/routes/session-attachments.ts +++ b/packages/control-plane/src/routes/session-attachments.ts @@ -52,7 +52,7 @@ import { GITHUB_SANDBOX_FALLBACK_ROUTE, GITHUB_USER_OR_SERVICE_ROUTE, json, - requirePermission, + requireSession, } from "./shared"; import { type SessionRouteContext, dispatchSession } from "./session-route"; @@ -243,13 +243,13 @@ sessionAttachmentRoutes.post( "/sessions/:id/attachments", admit({ ...GITHUB_USER_OR_SERVICE_ROUTE, - authorization: requirePermission("sessions.collaborate"), + authorization: requireSession("collaborate"), }), (c) => dispatchSession(c, handleAttachmentPost) ); sessionAttachmentRoutes.get( "/sessions/:id/attachments/:attachmentId", - admit({ ...GITHUB_SANDBOX_FALLBACK_ROUTE, authorization: requirePermission("sessions.read") }), + admit({ ...GITHUB_SANDBOX_FALLBACK_ROUTE, authorization: requireSession("read") }), (c) => dispatchSession(c, handleAttachmentGet) ); diff --git a/packages/control-plane/src/routes/session-batch-archive.ts b/packages/control-plane/src/routes/session-batch-archive.ts index db6769d887..b69c71c0a3 100644 --- a/packages/control-plane/src/routes/session-batch-archive.ts +++ b/packages/control-plane/src/routes/session-batch-archive.ts @@ -7,6 +7,7 @@ import { createLogger } from "../logger"; import { admit } from "../routing/admit"; import type { ControlPlaneHonoEnv } from "../routing/hono-env"; import { archiveSessionBatch } from "../session/batch-archive"; +import { evaluateSessionAdmission, teamsEnforcementMode } from "../authorization/session-admission"; import { parseBody } from "./body"; import type { SessionRuntimeClient } from "../session/runtime-client"; import { dispatchSession } from "./session-route"; @@ -31,7 +32,7 @@ sessionBatchArchiveRoutes.post( c, async ( request, - _env, + env, _params, ctx: UserRouteContext & { sessionRuntime: SessionRuntimeClient } ) => { @@ -41,13 +42,34 @@ sessionBatchArchiveRoutes.post( trace_id: ctx.trace_id, request_id: ctx.request_id, }); - const results = await archiveSessionBatch(body.sessionIds, ctx.sessionRuntime, log); + try { + teamsEnforcementMode(ctx, env); + } catch { + return json( + { error: "Authorization unavailable", code: "authorization_unavailable" }, + 503 + ); + } + const eligible: string[] = []; + const skipped: SessionBatchArchiveResponse["skipped"] = []; + for (const sessionId of body.sessionIds) { + const admission = await evaluateSessionAdmission(ctx, env, sessionId, "lifecycle", null); + if (admission.kind !== "allowed") { + skipped.push({ + sessionId, + reason: admission.kind === "not_found" ? "not_found" : "missing_permission", + }); + continue; + } + eligible.push(sessionId); + } + const results = await archiveSessionBatch(eligible, ctx.sessionRuntime, log); log.info("Session batch archive completed", { event: "session.batch_archive", user_id: ctx.principal.userId, results, }); - return json({ results } satisfies SessionBatchArchiveResponse); + return json({ results, skipped } satisfies SessionBatchArchiveResponse); } ) ); diff --git a/packages/control-plane/src/routes/session-child-spawn.ts b/packages/control-plane/src/routes/session-child-spawn.ts index 06c591619b..ea35d37d72 100644 --- a/packages/control-plane/src/routes/session-child-spawn.ts +++ b/packages/control-plane/src/routes/session-child-spawn.ts @@ -37,6 +37,7 @@ import { GITHUB_SANDBOX_FALLBACK_ROUTE, json, permissionRequirement, + sessionRequirement, requireAll, } from "./shared"; import { type SessionRouteContext, dispatchSession } from "./session-route"; @@ -380,8 +381,8 @@ sessionChildSpawnRoutes.post( admit({ ...GITHUB_SANDBOX_FALLBACK_ROUTE, authorization: requireAll( - permissionRequirement("sessions.create"), - permissionRequirement("sessions.collaborate") + sessionRequirement("collaborate"), + permissionRequirement("sessions.create") ), }), (c) => dispatchSession(c, handleSpawnChild) diff --git a/packages/control-plane/src/routes/session-children.ts b/packages/control-plane/src/routes/session-children.ts index e74df8dd03..1ebf1234c3 100644 --- a/packages/control-plane/src/routes/session-children.ts +++ b/packages/control-plane/src/routes/session-children.ts @@ -21,7 +21,9 @@ import { GITHUB_SANDBOX_FALLBACK_ROUTE, json, NO_AUTHORIZATION, - requirePermission, + requireAll, + requireSession, + sessionRequirement, SCM_AGNOSTIC_SANDBOX_ROUTE, type RequestContext, } from "./shared"; @@ -260,19 +262,25 @@ export const sessionChildRoutes = new Hono(); sessionChildRoutes.get( "/sessions/:id/children", - admit({ ...GITHUB_SANDBOX_FALLBACK_ROUTE, authorization: requirePermission("sessions.read") }), + admit({ ...GITHUB_SANDBOX_FALLBACK_ROUTE, authorization: requireSession("read") }), (c) => dispatch(c, handleListChildren) ); sessionChildRoutes.get( "/sessions/:id/children/:childId", - admit({ ...GITHUB_SANDBOX_FALLBACK_ROUTE, authorization: requirePermission("sessions.read") }), + admit({ + ...GITHUB_SANDBOX_FALLBACK_ROUTE, + authorization: requireAll(sessionRequirement("read"), sessionRequirement("read", "childId")), + }), (c) => dispatchSession(c, handleGetChild) ); sessionChildRoutes.post( "/sessions/:id/children/:childId/cancel", admit({ ...GITHUB_SANDBOX_FALLBACK_ROUTE, - authorization: requirePermission("sessions.lifecycle"), + authorization: requireAll( + sessionRequirement("read"), + sessionRequirement("lifecycle", "childId") + ), }), (c) => dispatchSession(c, handleCancelChild) ); diff --git a/packages/control-plane/src/routes/session-diffs.ts b/packages/control-plane/src/routes/session-diffs.ts index 86b2425e33..cff87e04e8 100644 --- a/packages/control-plane/src/routes/session-diffs.ts +++ b/packages/control-plane/src/routes/session-diffs.ts @@ -14,7 +14,7 @@ import { error, SCM_AGNOSTIC_SANDBOX_FALLBACK_ROUTE, SCM_AGNOSTIC_USER_OR_SERVICE_ROUTE, - requirePermission, + requireSession, } from "./shared"; import { type SessionRouteContext, dispatchSession } from "./session-route"; import type { Env } from "../types"; @@ -172,11 +172,11 @@ export const sessionDiffRoutes = new Hono(); const DIFF_READ = admit({ ...SCM_AGNOSTIC_USER_OR_SERVICE_ROUTE, - authorization: requirePermission("sessions.read"), + authorization: requireSession("read"), }); const DIFF_WRITE = admit({ ...SCM_AGNOSTIC_SANDBOX_FALLBACK_ROUTE, - authorization: requirePermission("sessions.collaborate"), + authorization: requireSession("collaborate"), }); sessionDiffRoutes.get("/sessions/:id/diff", DIFF_READ, (c) => dispatchSession(c, handleDiffState)); @@ -193,7 +193,7 @@ sessionDiffRoutes.post( "/sessions/:id/diff/retry", admit({ ...SCM_AGNOSTIC_USER_OR_SERVICE_ROUTE, - authorization: requirePermission("sessions.lifecycle"), + authorization: requireSession("lifecycle"), }), (c) => dispatchSession(c, handleDiffRetry) ); diff --git a/packages/control-plane/src/routes/session-export.test.ts b/packages/control-plane/src/routes/session-export.test.ts index dc2e3595dd..2eb2208851 100644 --- a/packages/control-plane/src/routes/session-export.test.ts +++ b/packages/control-plane/src/routes/session-export.test.ts @@ -20,6 +20,8 @@ import { createTestRequestHandler, TEST_BACKGROUND_TASK_CONTEXT, TEST_SERVICE_SECRETS, + sessionAdmissionStatement, + emptyStatement, } from "../router.test-support"; import type { PermissionId } from "@open-inspect/shared/rbac"; import type { ListSessionsForExportOptions, SessionExportRow } from "../db/session-export-store"; @@ -76,7 +78,10 @@ vi.mock("../logger", () => ({ const USER_PRINCIPAL: Principal = { kind: "user", userId: "user-1" }; function createEnv(permissions?: readonly PermissionId[]): Env { - const db = permissions ? authorizationDatabase({ permissions }) : authorizationDatabase(); + const db = authorizationDatabase({ + permissions, + statement: (sql) => sessionAdmissionStatement(sql) ?? emptyStatement(), + }); return createTestEnv({ ...TEST_SERVICE_SECRETS, DB: db, @@ -114,7 +119,7 @@ describe("GET /sessions/:id/export", () => { mocks.runtimeFetch.mockReset(); }); - it("refuses viewers before looking up the session", async () => { + it("refuses viewers without sessions.export after session admission", async () => { const response = await callExport( {}, { sessionId: "session-1", permissions: ["sessions.read"] } diff --git a/packages/control-plane/src/routes/session-export.ts b/packages/control-plane/src/routes/session-export.ts index 1d46164963..a2564b4fef 100644 --- a/packages/control-plane/src/routes/session-export.ts +++ b/packages/control-plane/src/routes/session-export.ts @@ -52,7 +52,14 @@ import type { SessionRuntimeClient } from "../session/runtime-client"; import type { Env } from "../types"; import { parseQuery } from "./query"; import { dispatchSession, type SessionRouteContext } from "./session-route"; -import { error, SCM_AGNOSTIC_USER_OR_SERVICE_ROUTE, requirePermission } from "./shared"; +import { + error, + SCM_AGNOSTIC_USER_OR_SERVICE_ROUTE, + requirePermission, + requireAll, + permissionRequirement, + sessionRequirement, +} from "./shared"; export const EXPORT_SCHEMA_VERSION = TRACE_EXPORT_SCHEMA_VERSION; const MAX_EXPORT_LIMIT = 500; @@ -350,6 +357,12 @@ const EXPORT_READ = admit({ export const sessionExportRoutes = new Hono(); sessionExportRoutes.get("/sessions/export", EXPORT_READ, (c) => dispatchSession(c, handleExport)); -sessionExportRoutes.get("/sessions/:id/export", EXPORT_READ, (c) => - dispatchSession(c, handleSingleExport) +sessionExportRoutes.get( + "/sessions/:id/export", + admit({ + ...SCM_AGNOSTIC_USER_OR_SERVICE_ROUTE, + authorization: requireAll(sessionRequirement("read"), permissionRequirement("sessions.export")), + cacheControl: "private, no-store", + }), + (c) => dispatchSession(c, handleSingleExport) ); diff --git a/packages/control-plane/src/routes/session-index.ts b/packages/control-plane/src/routes/session-index.ts index cebde869db..a8bbe4f4ce 100644 --- a/packages/control-plane/src/routes/session-index.ts +++ b/packages/control-plane/src/routes/session-index.ts @@ -25,6 +25,7 @@ import { json, SCM_AGNOSTIC_HUMAN_USER_ROUTE, requirePermission, + requireSession, type RequestContext, type UserRouteContext, } from "./shared"; @@ -284,11 +285,11 @@ sessionIndexRoutes.get( ); sessionIndexRoutes.patch( "/sessions/:id/read-state", - admit({ ...SCM_AGNOSTIC_HUMAN_USER_ROUTE, authorization: requirePermission("sessions.read") }), + admit({ ...SCM_AGNOSTIC_HUMAN_USER_ROUTE, authorization: requireSession("read") }), (c) => dispatch(c, handlePatchReadState) ); sessionIndexRoutes.delete( "/sessions/:id", - admit({ ...GITHUB_USER_OR_SERVICE_ROUTE, authorization: requirePermission("sessions.delete") }), + admit({ ...GITHUB_USER_OR_SERVICE_ROUTE, authorization: requireSession("delete") }), (c) => dispatch(c, handleDeleteSession) ); diff --git a/packages/control-plane/src/routes/session-media-stream.ts b/packages/control-plane/src/routes/session-media-stream.ts index f9bf6a6a2e..78e1f97b43 100644 --- a/packages/control-plane/src/routes/session-media-stream.ts +++ b/packages/control-plane/src/routes/session-media-stream.ts @@ -13,7 +13,7 @@ import { createStoredObjectResponse, } from "./responses/stored-object-response"; import { getSessionArtifactFromRuntime } from "./session-media-artifacts"; -import { error, GITHUB_USER_OR_SERVICE_ROUTE, requirePermission } from "./shared"; +import { error, GITHUB_USER_OR_SERVICE_ROUTE, requireSession } from "./shared"; import { type SessionRouteContext, dispatchSession } from "./session-route"; const logger = createLogger("router:session-media"); @@ -144,7 +144,7 @@ sessionMediaStreamRoutes.get( "/sessions/:id/media/:artifactId", admit({ ...GITHUB_USER_OR_SERVICE_ROUTE, - authorization: requirePermission("sessions.read", { + authorization: requireSession("read", { actorlessGrants: [{ service: "slack-bot" }], }), }), diff --git a/packages/control-plane/src/routes/session-media-upload.ts b/packages/control-plane/src/routes/session-media-upload.ts index b17d8e3613..b1881c5710 100644 --- a/packages/control-plane/src/routes/session-media-upload.ts +++ b/packages/control-plane/src/routes/session-media-upload.ts @@ -24,7 +24,7 @@ import { import type { ObjectStorage } from "../storage/object-storage"; import type { Env } from "../types"; import { listSessionArtifactsFromRuntime, persistMediaArtifact } from "./session-media-artifacts"; -import { error, GITHUB_SANDBOX_FALLBACK_ROUTE, json, requirePermission } from "./shared"; +import { error, GITHUB_SANDBOX_FALLBACK_ROUTE, json, requireSession } from "./shared"; import { type SessionRouteContext, dispatchSession } from "./session-route"; function getRequiredFormString(value: MultipartFieldValue | null, name: string): string | Response { @@ -247,7 +247,7 @@ sessionMediaUploadRoutes.post( "/sessions/:id/media", admit({ ...GITHUB_SANDBOX_FALLBACK_ROUTE, - authorization: requirePermission("sessions.collaborate"), + authorization: requireSession("collaborate"), }), (c) => dispatchSession(c, handleMediaUpload) ); diff --git a/packages/control-plane/src/routes/session-prompt.ts b/packages/control-plane/src/routes/session-prompt.ts index 1331a5f44c..e8c8f618e7 100644 --- a/packages/control-plane/src/routes/session-prompt.ts +++ b/packages/control-plane/src/routes/session-prompt.ts @@ -28,7 +28,7 @@ import { type GitHubEnrichment, } from "../session/identity"; import type { Env } from "../types"; -import { error, json, GITHUB_USER_OR_SERVICE_ROUTE, requirePermission } from "./shared"; +import { error, json, GITHUB_USER_OR_SERVICE_ROUTE, requireSession } from "./shared"; import { parseJsonBody } from "./body"; import { type SessionRouteContext, dispatchSession } from "./session-route"; @@ -175,7 +175,7 @@ sessionPromptRoutes.post( "/sessions/:id/prompt", admit({ ...GITHUB_USER_OR_SERVICE_ROUTE, - authorization: requirePermission("sessions.collaborate"), + authorization: requireSession("collaborate"), }), (c) => dispatchSession(c, handleSessionPrompt) ); diff --git a/packages/control-plane/src/routes/session-pull-requests.ts b/packages/control-plane/src/routes/session-pull-requests.ts index b49594a598..c91cc0222e 100644 --- a/packages/control-plane/src/routes/session-pull-requests.ts +++ b/packages/control-plane/src/routes/session-pull-requests.ts @@ -3,7 +3,7 @@ import { admit } from "../routing/admit"; import type { ControlPlaneHonoEnv } from "../routing/hono-env"; import { SessionInternalPaths } from "../session/contracts"; import type { Env } from "../types"; -import { GITHUB_USER_OR_SERVICE_ROUTE, requirePermission } from "./shared"; +import { GITHUB_USER_OR_SERVICE_ROUTE, requireSession } from "./shared"; import { type SessionRouteContext, dispatchSession } from "./session-route"; /** @@ -31,7 +31,7 @@ sessionPullRequestRoutes.post( "/sessions/:id/pull-requests/refresh", admit({ ...GITHUB_USER_OR_SERVICE_ROUTE, - authorization: requirePermission("sessions.lifecycle"), + authorization: requireSession("lifecycle"), }), (c) => dispatchSession(c, handleRefreshPullRequests) ); diff --git a/packages/control-plane/src/routes/session-route.ts b/packages/control-plane/src/routes/session-route.ts index 1b21c063ef..7fc5c8fbeb 100644 --- a/packages/control-plane/src/routes/session-route.ts +++ b/packages/control-plane/src/routes/session-route.ts @@ -13,7 +13,7 @@ export function withSessionRuntime( env: Env, ctx: Context ): Context & { sessionRuntime: SessionRuntimeClient } { - return { ...ctx, sessionRuntime: createSessionRuntimeClient(env, ctx) }; + return Object.assign(ctx, { sessionRuntime: createSessionRuntimeClient(env, ctx) }); } /** Run a session handler for an admitted request, with the runtime client bound to it. */ diff --git a/packages/control-plane/src/routes/session-runtime-proxy.test.ts b/packages/control-plane/src/routes/session-runtime-proxy.test.ts index b2ce928f21..1ca6cdff4b 100644 --- a/packages/control-plane/src/routes/session-runtime-proxy.test.ts +++ b/packages/control-plane/src/routes/session-runtime-proxy.test.ts @@ -11,6 +11,7 @@ import { TEST_SERVICE_SECRETS, createTestRequestHandler, fakeSessionRuntimeDispatch, + TEST_SESSION_ROW, } from "../router.test-support"; import { SessionInternalPaths } from "../session/contracts"; import type { Env } from "../types"; @@ -34,6 +35,7 @@ const SANDBOX_TOKEN_HEADERS = { Authorization: "Bearer sandbox-token" }; type DatabaseOptions = { /** Custom-role grants for user-1; omitted means the owner role with every permission. */ permissions?: PermissionId[]; + visibility?: "private"; /** Answers every statement admission and the proxy's own reads do not own. */ delegate?: SqlDatabase; }; @@ -50,9 +52,15 @@ function createDatabase(options: DatabaseOptions = {}): SqlDatabase { if (sql.includes("FROM role_permissions")) { return (options.permissions ?? []).map((permission_id) => ({ permission_id })); } + if (sql.includes("FROM team_memberships") || sql.includes("FROM session_collaborators")) + return []; return null; }; const row = (sql: string): unknown => { + if (sql.includes("SELECT * FROM sessions")) + return options.visibility === "private" + ? { ...TEST_SESSION_ROW, visibility: "private", user_id: "another-user" } + : TEST_SESSION_ROW; if (sql.includes("FROM users u")) return { user_id: "user-1", suspended_at: null, ...role }; if (sql.includes("FROM session_model_provider_auth")) { return { @@ -205,54 +213,73 @@ describe("session runtime proxy routes", () => { }); it.each([ - { permissions: ["sessions.read"] as PermissionId[], exposed: false }, + { + permissions: ["sessions.read"] as PermissionId[], + exposed: false, + mode: "shadow", + visibility: "workspace", + }, { permissions: ["sessions.read", "sessions.sandbox_access"] as PermissionId[], exposed: true, + mode: "shadow", + visibility: "workspace", }, - ])("scopes snapshot sandbox locations to sandbox access ($exposed)", async (input) => { - const fetch = vi.fn(async () => - Response.json({ - session: { - id: "session-1", - title: "Session", - repoOwner: "acme", - repoName: "web", - baseBranch: "main", - branchName: "feature", - status: "active", - sandboxStatus: "ready", - messageCount: 0, - createdAt: 1, - codeServerUrl: "https://code.example", - vncUrl: "https://vnc.example", - ttydUrl: "https://terminal.example", - tunnelUrls: { "3000": "https://app.example" }, - sandboxDashboardUrl: "https://provider.example", - }, - artifacts: [], - promptQueue: [], - timeline: { events: [], hasMore: false, cursor: null }, - }) - ); + ...(["off", "shadow", "on"] as const).map((mode) => ({ + permissions: undefined, + exposed: false, + mode, + visibility: "private" as const, + })), + ])( + "scopes $mode $visibility snapshot sandbox locations to sandbox access ($exposed)", + async (input) => { + const fetch = vi.fn(async () => + Response.json({ + session: { + id: "session-1", + title: "Session", + repoOwner: "acme", + repoName: "web", + baseBranch: "main", + branchName: "feature", + status: "active", + sandboxStatus: "ready", + messageCount: 0, + createdAt: 1, + codeServerUrl: "https://code.example", + vncUrl: "https://vnc.example", + ttydUrl: "https://terminal.example", + tunnelUrls: { "3000": "https://app.example" }, + sandboxDashboardUrl: "https://provider.example", + }, + artifacts: [], + promptQueue: [], + timeline: { events: [], hasMore: false, cursor: null }, + }) + ); - const response = await dispatch( - new Request("https://test.local/sessions/session-1"), - createEnv(fetch, { permissions: input.permissions }) - ); - const snapshot = (await response.json()) as { session: Record }; + const response = await dispatch(new Request("https://test.local/sessions/session-1"), { + ...createEnv(fetch, { + permissions: input.permissions, + visibility: input.visibility === "private" ? "private" : undefined, + }), + TEAMS_ENFORCEMENT: input.mode, + }); + const snapshot = (await response.json()) as { session: Record }; - expect(response.status).toBe(200); - if (input.exposed) { - expect(snapshot.session).toHaveProperty("codeServerUrl", "https://code.example"); - } else { - expect(snapshot.session).not.toHaveProperty("codeServerUrl"); - expect(snapshot.session).not.toHaveProperty("vncUrl"); - expect(snapshot.session).not.toHaveProperty("ttydUrl"); - expect(snapshot.session).not.toHaveProperty("tunnelUrls"); - expect(snapshot.session).not.toHaveProperty("sandboxDashboardUrl"); + expect(response.status).toBe(200); + if (input.exposed) { + expect(snapshot.session).toHaveProperty("codeServerUrl", "https://code.example"); + } else { + expect(snapshot.session).not.toHaveProperty("codeServerUrl"); + expect(snapshot.session).not.toHaveProperty("vncUrl"); + expect(snapshot.session).not.toHaveProperty("ttydUrl"); + expect(snapshot.session).not.toHaveProperty("tunnelUrls"); + expect(snapshot.session).not.toHaveProperty("sandboxDashboardUrl"); + } } - }); + ); it("forwards event query strings through the session runtime dependency", async () => { const requests: Request[] = []; @@ -729,7 +756,12 @@ describe("session runtime proxy routes", () => { }); it("rejects a malformed budget body before reading the session", async () => { - const get = vi.spyOn(SessionIndexStore.prototype, "get"); + const get = vi.spyOn(SessionIndexStore.prototype, "get").mockResolvedValue({ + id: "session-1", + userId: "user-1", + ownerTeamId: null, + visibility: "workspace", + } as Awaited>); const fetch = vi.fn(async () => Response.json({ maxSessionCostUsd: 20 })); const response = await dispatch( @@ -743,7 +775,7 @@ describe("session runtime proxy routes", () => { expect(response.status).toBe(400); await expect(response.json()).resolves.toEqual({ error: "Invalid budget request" }); - expect(get).not.toHaveBeenCalled(); + expect(get).toHaveBeenCalledOnce(); expect(fetch).not.toHaveBeenCalled(); }); }); diff --git a/packages/control-plane/src/routes/session-runtime-proxy.ts b/packages/control-plane/src/routes/session-runtime-proxy.ts index d0e94129f1..b77f2e4ff1 100644 --- a/packages/control-plane/src/routes/session-runtime-proxy.ts +++ b/packages/control-plane/src/routes/session-runtime-proxy.ts @@ -15,6 +15,7 @@ import { import { z } from "zod"; import { UserStore } from "../db/user-store"; import { SessionIndexStore } from "../db/session-index"; +import { checkSessionAccess } from "@open-inspect/shared"; import type { SubscriptionProviderId } from "@open-inspect/shared/types/provider-accounts"; import { SessionInternalPaths, type SessionInternalPath } from "../session/contracts"; import type { Env } from "../types"; @@ -23,7 +24,7 @@ import { GITHUB_SANDBOX_FALLBACK_ROUTE, GITHUB_USER_OR_SERVICE_ROUTE, NO_AUTHORIZATION, - requirePermission, + requireSession, SCM_AGNOSTIC_SANDBOX_FALLBACK_ROUTE, SCM_AGNOSTIC_HANDLER_AUTHENTICATED_ROUTE, SCM_AGNOSTIC_SANDBOX_ROUTE, @@ -174,9 +175,12 @@ async function handleSessionSnapshot( const parsed = sessionSnapshotSchema.safeParse(await response.json().catch(() => null)); if (!parsed.success) return error("Invalid session snapshot", 502); - const snapshot = ctx.authorization?.permissions.includes("sessions.sandbox_access") - ? parsed.data - : redactSessionSnapshotSandboxAccess(parsed.data); + const admission = ctx.sessionAdmission; + const sandboxAllowed = + admission && (admission.row.visibility === "private" || ctx.teamsEnforcementMode === "on") + ? checkSessionAccess(admission.viewer, admission.row, "sandbox").allowed + : ctx.authorization?.permissions.includes("sessions.sandbox_access"); + const snapshot = sandboxAllowed ? parsed.data : redactSessionSnapshotSandboxAccess(parsed.data); const headers = new Headers(response.headers); headers.delete("Content-Length"); return Response.json(snapshot, { headers }); @@ -305,7 +309,7 @@ async function handleBudgetUpdate( /** Every proxied session operation, by the name its route is known by. */ const LIFECYCLE = admit({ ...GITHUB_USER_OR_SERVICE_ROUTE, - authorization: requirePermission("sessions.lifecycle"), + authorization: requireSession("lifecycle"), }); export const sessionRuntimeProxyRoutes = new Hono(); @@ -314,20 +318,20 @@ sessionRuntimeProxyRoutes.get( "/sessions/:id/sandbox-access", admit({ ...SCM_AGNOSTIC_HUMAN_USER_ROUTE, - authorization: requirePermission("sessions.sandbox_access"), + authorization: requireSession("sandbox"), }), (c) => dispatchSession(c, simpleProxy({ internalPath: SessionInternalPaths.sandboxAccess })) ); sessionRuntimeProxyRoutes.get( "/sessions/:id", - admit({ ...SCM_AGNOSTIC_HUMAN_USER_ROUTE, authorization: requirePermission("sessions.read") }), + admit({ ...SCM_AGNOSTIC_HUMAN_USER_ROUTE, authorization: requireSession("read") }), (c) => dispatchSession(c, handleSessionSnapshot) ); sessionRuntimeProxyRoutes.post( "/sessions/:id/stop", admit({ ...GITHUB_USER_OR_SERVICE_ROUTE, - authorization: requirePermission("sessions.lifecycle", { + authorization: requireSession("lifecycle", { actorlessGrants: [{ service: "linear-bot" }], }), }), @@ -346,7 +350,7 @@ sessionRuntimeProxyRoutes.get( "/sessions/:id/events", admit({ ...GITHUB_USER_OR_SERVICE_ROUTE, - authorization: requirePermission("sessions.read", { + authorization: requireSession("read", { actorlessGrants: [{ service: "slack-bot" }, { service: "linear-bot" }], }), }), @@ -360,7 +364,7 @@ sessionRuntimeProxyRoutes.get( "/sessions/:id/artifacts", admit({ ...GITHUB_USER_OR_SERVICE_ROUTE, - authorization: requirePermission("sessions.read", { + authorization: requireSession("read", { actorlessGrants: [{ service: "slack-bot" }, { service: "linear-bot" }], }), }), @@ -368,20 +372,20 @@ sessionRuntimeProxyRoutes.get( ); sessionRuntimeProxyRoutes.get( "/sessions/:id/participants", - admit({ ...GITHUB_USER_OR_SERVICE_ROUTE, authorization: requirePermission("sessions.read") }), + admit({ ...GITHUB_USER_OR_SERVICE_ROUTE, authorization: requireSession("read") }), (c) => dispatchSession(c, simpleProxy({ internalPath: SessionInternalPaths.participants })) ); sessionRuntimeProxyRoutes.get( "/sessions/:id/participant-profiles", admit({ ...SCM_AGNOSTIC_USER_OR_SERVICE_ROUTE, - authorization: requirePermission("sessions.read"), + authorization: requireSession("read"), }), (c) => dispatchSession(c, handleParticipantProfiles) ); sessionRuntimeProxyRoutes.get( "/sessions/:id/messages", - admit({ ...GITHUB_USER_OR_SERVICE_ROUTE, authorization: requirePermission("sessions.read") }), + admit({ ...GITHUB_USER_OR_SERVICE_ROUTE, authorization: requireSession("read") }), (c) => dispatchSession( c, @@ -392,7 +396,7 @@ sessionRuntimeProxyRoutes.post( "/sessions/:id/pr", admit({ ...GITHUB_SANDBOX_FALLBACK_ROUTE, - authorization: requirePermission("sessions.collaborate"), + authorization: requireSession("collaborate"), }), (c) => dispatchSession(c, handleCreatePR) ); @@ -419,7 +423,7 @@ sessionRuntimeProxyRoutes.get( "/sessions/:id/tunnel-urls", admit({ ...SCM_AGNOSTIC_SANDBOX_FALLBACK_ROUTE, - authorization: requirePermission("sessions.sandbox_access"), + authorization: requireSession("sandbox"), }), (c) => dispatchSession( @@ -440,7 +444,7 @@ sessionRuntimeProxyRoutes.patch( "/sessions/:id/budget", admit({ ...SCM_AGNOSTIC_HUMAN_USER_ROUTE, - authorization: requirePermission("sessions.lifecycle"), + authorization: requireSession("lifecycle"), }), (c) => dispatchSession(c, handleBudgetUpdate) ); diff --git a/packages/control-plane/src/routes/session-skills.ts b/packages/control-plane/src/routes/session-skills.ts index b8488a31d6..40167d07ff 100644 --- a/packages/control-plane/src/routes/session-skills.ts +++ b/packages/control-plane/src/routes/session-skills.ts @@ -9,7 +9,7 @@ import { error, json, NO_AUTHORIZATION, - requirePermission, + requireSession, SCM_AGNOSTIC_SANDBOX_ROUTE, SCM_AGNOSTIC_HUMAN_USER_ROUTE, type SandboxRouteContext, @@ -92,7 +92,7 @@ export const sessionSkillRoutes = new Hono(); sessionSkillRoutes.get( "/sessions/:id/skills", - admit({ ...SCM_AGNOSTIC_HUMAN_USER_ROUTE, authorization: requirePermission("sessions.read") }), + admit({ ...SCM_AGNOSTIC_HUMAN_USER_ROUTE, authorization: requireSession("read") }), (c) => dispatch(c, handleSessionSkillsView) ); diff --git a/packages/control-plane/src/routes/session-ws-token.ts b/packages/control-plane/src/routes/session-ws-token.ts index 79361045a1..d1a41e6c55 100644 --- a/packages/control-plane/src/routes/session-ws-token.ts +++ b/packages/control-plane/src/routes/session-ws-token.ts @@ -2,10 +2,9 @@ import { Hono } from "hono"; import { admit } from "../routing/admit"; import type { ControlPlaneHonoEnv } from "../routing/hono-env"; import { applyIdentityEnforcement } from "../routing/identity-enforcement"; -import { SESSION_WEBSOCKET_CONNECT_PERMISSION } from "@open-inspect/shared/rbac"; import { SessionInternalPaths, sessionScmDisplayFieldsSchema } from "../session/contracts"; import type { Env } from "../types"; -import { error, GITHUB_USER_OR_SERVICE_ROUTE, requirePermission } from "./shared"; +import { error, GITHUB_USER_OR_SERVICE_ROUTE, requireSession } from "./shared"; import { parseJsonBody } from "./body"; import { dispatchSession, type SessionRouteContext } from "./session-route"; @@ -55,7 +54,7 @@ sessionWsTokenRoutes.post( "/sessions/:id/ws-token", admit({ ...GITHUB_USER_OR_SERVICE_ROUTE, - authorization: requirePermission(SESSION_WEBSOCKET_CONNECT_PERMISSION), + authorization: requireSession("read"), }), (c) => dispatchSession(c, handleSessionWsToken) ); diff --git a/packages/control-plane/src/routes/shared.ts b/packages/control-plane/src/routes/shared.ts index 94c369b063..90c8b8225f 100644 --- a/packages/control-plane/src/routes/shared.ts +++ b/packages/control-plane/src/routes/shared.ts @@ -10,6 +10,7 @@ import type { Logger } from "../logger"; import type { PermissionId } from "@open-inspect/shared/rbac"; import type { ServiceName } from "@open-inspect/shared/service-auth"; import type { TeamCapabilities } from "@open-inspect/shared/types/team-access"; +import type { SessionAction } from "@open-inspect/shared"; import { createSourceControlProviderFromEnv, SourceControlProviderError, @@ -45,7 +46,8 @@ export type RouteAuthorizationRequirement = operation: "manage" | "trigger"; automationIdParam: string; } - | { kind: "team"; teamIdParam: string; need: keyof TeamCapabilities | "read" }; + | { kind: "team"; teamIdParam: string; need: keyof TeamCapabilities | "read" } + | { kind: "session"; sessionIdParam: string; action: SessionAction }; type BotServiceName = Exclude; const DEFAULT_AUDIT_ALLOWED = false; @@ -141,6 +143,7 @@ const AUDITED_ALLOWED_PERMISSIONS = new Set([ ]); function auditsAllowedRequirement(requirement: RouteAuthorizationRequirement): boolean { + if (requirement.kind === "session") return requirement.action !== "read"; if (requirement.kind === "permission") { return AUDITED_ALLOWED_PERMISSIONS.has(requirement.permission); } @@ -193,6 +196,25 @@ export function requireTeam( }; } +export function sessionRequirement( + action: SessionAction, + sessionIdParam = "id" +): RouteAuthorizationRequirement { + return { kind: "session", sessionIdParam, action }; +} + +export function requireSession( + action: SessionAction, + options?: { sessionIdParam?: string; actorlessGrants?: readonly ActorlessServiceGrant[] } +): RouteAuthorization { + return { + kind: "active-user", + allOf: [sessionRequirement(action, options?.sessionIdParam)], + service: { kind: "actor", actorlessGrants: options?.actorlessGrants }, + auditAllowed: action !== "read", + }; +} + /** Require an active user to satisfy every supplied authorization requirement. */ export function requireAll(...allOf: readonly RouteAuthorizationRequirement[]): RouteAuthorization { return { diff --git a/packages/control-plane/src/routes/slack-notify.ts b/packages/control-plane/src/routes/slack-notify.ts index 91515061d8..368a56e1de 100644 --- a/packages/control-plane/src/routes/slack-notify.ts +++ b/packages/control-plane/src/routes/slack-notify.ts @@ -20,12 +20,7 @@ import { IntegrationSettingsStore, resolveSlackSettings } from "../db/integratio import { SessionIndexStore } from "../db/session-index"; import { createLogger } from "../logger"; import type { Env } from "../types"; -import { - GITHUB_SANDBOX_FALLBACK_ROUTE, - json, - requirePermission, - type RequestContext, -} from "./shared"; +import { GITHUB_SANDBOX_FALLBACK_ROUTE, json, requireSession, type RequestContext } from "./shared"; const logger = createLogger("slack-notify"); @@ -317,7 +312,7 @@ slackNotifyRoutes.post( "/sessions/:id/slack-notify", admit({ ...GITHUB_SANDBOX_FALLBACK_ROUTE, - authorization: requirePermission("sessions.collaborate"), + authorization: requireSession("collaborate"), }), (c) => dispatch(c, handleSlackNotify) ); diff --git a/packages/control-plane/src/routing/hono-app.ts b/packages/control-plane/src/routing/hono-app.ts index 9cdb99a170..8dc9b2650c 100644 --- a/packages/control-plane/src/routing/hono-app.ts +++ b/packages/control-plane/src/routing/hono-app.ts @@ -179,13 +179,18 @@ export function createControlPlaneApp( } if (!unexpected) logRequest(c.res, context, method, pathname, startedAt); - if (shouldAuditAllowedDecision(result.decision)) { + const decision = { + ...result.decision, + ...(context.shadowSessionDenial ? { shadowReason: context.shadowSessionDenial } : {}), + ...(context.shadowBatchDenials?.length ? { shadowDenials: context.shadowBatchDenials } : {}), + }; + if (shouldAuditAllowedDecision(decision)) { await auditRouteAuthorizationDecision({ ctx: context, method, path: pathname, response: c.res, - decision: result.decision, + decision, }); } replaceResponse(c, finalizeRouteResponse(c.res, policy, context)); diff --git a/packages/control-plane/src/routing/route-admission.ts b/packages/control-plane/src/routing/route-admission.ts index 705fbab056..a9d3779ae4 100644 --- a/packages/control-plane/src/routing/route-admission.ts +++ b/packages/control-plane/src/routing/route-admission.ts @@ -13,6 +13,8 @@ import type { } from "../authorization/request-audit"; import { AuthorizationError, AuthorizationService } from "../authorization/service"; import { serviceAllowsPermission } from "../authorization/service-permissions"; +import { evaluateSessionAdmission, viewerFromContext } from "../authorization/session-admission"; +import { legacyPermissionForAction } from "../authorization/teams-enforcement"; import { AutomationStore } from "../db/automation-store"; import { TeamStore } from "../db/teams"; import { TeamMembershipStore } from "../db/team-memberships"; @@ -343,7 +345,12 @@ function enforceStaticServicePermissionCeiling( if (policy.authorization.kind !== "active-user") return null; for (const requirement of policy.authorization.allOf) { - const permission = requirement.kind === "permission" ? requirement.permission : null; + const permission = + requirement.kind === "permission" + ? requirement.permission + : requirement.kind === "session" + ? legacyPermissionForAction(requirement.action) + : null; if (permission && !serviceAllowsPermission(principal.service, permission)) { return authorizationDenial( json({ error: "Forbidden", code: "service_capability_required" }, 403), @@ -602,23 +609,24 @@ async function enforceTeamRequirement( const teamId = params[requirement.teamIdParam]; if (!teamId) return { response: json({ error: "Invalid team route" }, 400) }; try { - const authorization = ctx.authorization; - if (!authorization) throw new Error("Missing request authorization"); const team = await new TeamStore(ctx.db).getById(teamId); if (!team) return { response: error("Team not found", 404) }; const memberships = new TeamMembershipStore(ctx.db); - const viewerMemberships = await memberships.listForUser(ctx.principal.userId); + const viewer = viewerFromContext( + ctx, + (ctx.sessionMemberships ??= await memberships.listForUser(ctx.principal.userId)) + ); + if (viewer.kind !== "user") throw new Error("Missing team viewer"); const access = resolveTeamAccess( { - userId: ctx.principal.userId, - roleKey: authorization.role.key, - memberships: viewerMemberships, + userId: viewer.userId, + roleKey: viewer.roleKey, + memberships: viewer.memberships, }, { ...team, leadCount: await memberships.countLeads(teamId) } ); - const isAdmin = - authorization.role.key === "owner" || authorization.role.key === "administrator"; - const visible = isAdmin || viewerMemberships.has(teamId); + const isAdmin = viewer.roleKey === "owner" || viewer.roleKey === "administrator"; + const visible = isAdmin || viewer.memberships.has(teamId); if (!visible && requirement.need !== "canJoin") return { response: error("Team not found", 404) }; if (requirement.need !== "read" && !access[requirement.need]) { @@ -646,6 +654,59 @@ async function enforceTeamRequirement( } } +async function enforceSessionRequirement( + requirement: Extract, + params: RouteParams, + env: Env, + ctx: RequestContext, + evidence: AuthorizationEvidence +): Promise { + const sessionId = params[requirement.sessionIdParam]; + if (!sessionId) return { response: json({ error: "Invalid session route" }, 400) }; + try { + const result = await evaluateSessionAdmission( + ctx, + env, + sessionId, + requirement.action, + requirement.sessionIdParam === "childId" ? "child" : "session" + ); + if (result.kind === "not_found") { + return authorizationDenial( + error("Session not found", 404), + evidence, + requirement, + "session_not_visible", + "Session not found" + ); + } + if (result.kind === "action_denied") { + return authorizationDenial( + json( + { error: "Forbidden", code: "session_action_denied", reason_code: result.reason }, + 403 + ), + evidence, + requirement, + result.reason, + "Forbidden" + ); + } + if (result.legacyPermission) { + const legacy = await enforcePermissionRequirement( + { kind: "permission", permission: result.legacyPermission }, + ctx, + evidence + ); + if (legacy) return legacy; + } + evidence.requirements.push(requirement); + return null; + } catch { + return authorizationUnavailable(); + } +} + function allowed( policy: RouteAdmissionPolicy, admission: AllowedAuthorizationDecision["admission"], @@ -721,6 +782,9 @@ async function enforceRouteAuthorization( case "team": failure = await enforceTeamRequirement(requirement, params, ctx, evidence); break; + case "session": + failure = await enforceSessionRequirement(requirement, params, env, ctx, evidence); + break; } if (failure) return resultForFailure(failure); } diff --git a/packages/control-plane/src/types.ts b/packages/control-plane/src/types.ts index d1f082adce..83fe530a1d 100644 --- a/packages/control-plane/src/types.ts +++ b/packages/control-plane/src/types.ts @@ -98,6 +98,7 @@ export interface EnvConfig { SANDBOX_BOOT_TIMEOUT_MS?: string; // Longest a connected sandbox may boot before it is failed, in ms; defaults to DEFAULT_BOOT_BUDGET_CONFIG EXECUTION_TIMEOUT_MS?: string; // Max processing time for one message before auto-fail, for sessions and for the automation runs watching them; overridden per session by sandboxTimeoutMs, and falls back to DEFAULT_SANDBOX_TIMEOUT_SECONDS SECRETS_CAP_ENFORCEMENT?: string; // "enforce" (default) fails spawn/build on oversized secret payloads; set "warn" to only log + TEAMS_ENFORCEMENT?: string; // "off" | "shadow" (default) | "on" // Logging LOG_LEVEL?: string; // "debug" | "info" | "warn" | "error" (default: "info") diff --git a/packages/control-plane/test/integration/__snapshots__/hono-route-catalog-conformance.test.ts.snap b/packages/control-plane/test/integration/__snapshots__/hono-route-catalog-conformance.test.ts.snap index 76c0a719e7..306b0ae1ea 100644 --- a/packages/control-plane/test/integration/__snapshots__/hono-route-catalog-conformance.test.ts.snap +++ b/packages/control-plane/test/integration/__snapshots__/hono-route-catalog-conformance.test.ts.snap @@ -24,49 +24,49 @@ exports[`Hono route catalog conformance > dispatches every frozen method/path/po "{"identity":"POST /sessions","pathname":"/sessions","groups":{},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.create"}],"auditAllowed":true,"service":{"kind":"actor"}},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":true}", "{"identity":"GET /sessions","pathname":"/sessions","groups":{},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.read"}],"auditAllowed":false,"service":{"kind":"actor"}},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", "{"identity":"GET /sessions/inbox","pathname":"/sessions/inbox","groups":{},"authentication":"user","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.read"}],"auditAllowed":false,"service":{"kind":"deny"}},"supportedScmProviders":"all","cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"PATCH /sessions/:id/read-state","pathname":"/sessions/fixture-22-id%2Fraw/read-state","groups":{"id":"fixture-22-id%2Fraw"},"authentication":"user","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.read"}],"auditAllowed":false,"service":{"kind":"actor"}},"supportedScmProviders":"all","cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"DELETE /sessions/:id","pathname":"/sessions/fixture-23-id%2Fraw","groups":{"id":"fixture-23-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.delete"}],"auditAllowed":true,"service":{"kind":"actor"}},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"PATCH /sessions/:id/read-state","pathname":"/sessions/fixture-22-id%2Fraw/read-state","groups":{"id":"fixture-22-id%2Fraw"},"authentication":"user","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"read"}],"service":{"kind":"actor"},"auditAllowed":false},"supportedScmProviders":"all","cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"DELETE /sessions/:id","pathname":"/sessions/fixture-23-id%2Fraw","groups":{"id":"fixture-23-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"delete"}],"service":{"kind":"actor"},"auditAllowed":true},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", "{"identity":"GET /sessions/export","pathname":"/sessions/export","groups":{},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.export"}],"auditAllowed":true,"service":{"kind":"actor"}},"supportedScmProviders":"all","cacheControl":"private, no-store","hasServiceActorClaims":false}", - "{"identity":"GET /sessions/:id/export","pathname":"/sessions/fixture-25-id%2Fraw/export","groups":{"id":"fixture-25-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.export"}],"auditAllowed":true,"service":{"kind":"actor"}},"supportedScmProviders":"all","cacheControl":"private, no-store","hasServiceActorClaims":false}", - "{"identity":"GET /sessions/:id/sandbox-access","pathname":"/sessions/fixture-26-id%2Fraw/sandbox-access","groups":{"id":"fixture-26-id%2Fraw"},"authentication":"user","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.sandbox_access"}],"auditAllowed":true,"service":{"kind":"actor"}},"supportedScmProviders":"all","cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"GET /sessions/:id","pathname":"/sessions/fixture-27-id%2Fraw","groups":{"id":"fixture-27-id%2Fraw"},"authentication":"user","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.read"}],"auditAllowed":false,"service":{"kind":"actor"}},"supportedScmProviders":"all","cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"POST /sessions/:id/stop","pathname":"/sessions/fixture-28-id%2Fraw/stop","groups":{"id":"fixture-28-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.lifecycle"}],"auditAllowed":true,"service":{"kind":"actor","actorlessGrants":[{"service":"linear-bot"}]}},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"GET /sessions/:id/export","pathname":"/sessions/fixture-25-id%2Fraw/export","groups":{"id":"fixture-25-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"read"},{"kind":"permission","permission":"sessions.export"}],"service":{"kind":"actor"},"auditAllowed":true},"supportedScmProviders":"all","cacheControl":"private, no-store","hasServiceActorClaims":false}", + "{"identity":"GET /sessions/:id/sandbox-access","pathname":"/sessions/fixture-26-id%2Fraw/sandbox-access","groups":{"id":"fixture-26-id%2Fraw"},"authentication":"user","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"sandbox"}],"service":{"kind":"actor"},"auditAllowed":true},"supportedScmProviders":"all","cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"GET /sessions/:id","pathname":"/sessions/fixture-27-id%2Fraw","groups":{"id":"fixture-27-id%2Fraw"},"authentication":"user","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"read"}],"service":{"kind":"actor"},"auditAllowed":false},"supportedScmProviders":"all","cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"POST /sessions/:id/stop","pathname":"/sessions/fixture-28-id%2Fraw/stop","groups":{"id":"fixture-28-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"lifecycle"}],"service":{"kind":"actor","actorlessGrants":[{"service":"linear-bot"}]},"auditAllowed":true},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", "{"identity":"POST /sessions/:id/sandbox-error","pathname":"/sessions/fixture-29-id%2Fraw/sandbox-error","groups":{"id":"fixture-29-id%2Fraw"},"authentication":"handler-authenticated","authorization":{"kind":"none","auditAllowed":false},"supportedScmProviders":"all","cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"GET /sessions/:id/events","pathname":"/sessions/fixture-30-id%2Fraw/events","groups":{"id":"fixture-30-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.read"}],"auditAllowed":false,"service":{"kind":"actor","actorlessGrants":[{"service":"slack-bot"},{"service":"linear-bot"}]}},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"GET /sessions/:id/artifacts","pathname":"/sessions/fixture-31-id%2Fraw/artifacts","groups":{"id":"fixture-31-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.read"}],"auditAllowed":false,"service":{"kind":"actor","actorlessGrants":[{"service":"slack-bot"},{"service":"linear-bot"}]}},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"GET /sessions/:id/participants","pathname":"/sessions/fixture-32-id%2Fraw/participants","groups":{"id":"fixture-32-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.read"}],"auditAllowed":false,"service":{"kind":"actor"}},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"GET /sessions/:id/participant-profiles","pathname":"/sessions/fixture-33-id%2Fraw/participant-profiles","groups":{"id":"fixture-33-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.read"}],"auditAllowed":false,"service":{"kind":"actor"}},"supportedScmProviders":"all","cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"GET /sessions/:id/messages","pathname":"/sessions/fixture-34-id%2Fraw/messages","groups":{"id":"fixture-34-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.read"}],"auditAllowed":false,"service":{"kind":"actor"}},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"POST /sessions/:id/pr","pathname":"/sessions/fixture-35-id%2Fraw/pr","groups":{"id":"fixture-35-id%2Fraw"},"authentication":"user-or-service-with-sandbox-fallback","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.collaborate"}],"auditAllowed":true,"service":{"kind":"actor"}},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"GET /sessions/:id/events","pathname":"/sessions/fixture-30-id%2Fraw/events","groups":{"id":"fixture-30-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"read"}],"service":{"kind":"actor","actorlessGrants":[{"service":"slack-bot"},{"service":"linear-bot"}]},"auditAllowed":false},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"GET /sessions/:id/artifacts","pathname":"/sessions/fixture-31-id%2Fraw/artifacts","groups":{"id":"fixture-31-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"read"}],"service":{"kind":"actor","actorlessGrants":[{"service":"slack-bot"},{"service":"linear-bot"}]},"auditAllowed":false},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"GET /sessions/:id/participants","pathname":"/sessions/fixture-32-id%2Fraw/participants","groups":{"id":"fixture-32-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"read"}],"service":{"kind":"actor"},"auditAllowed":false},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"GET /sessions/:id/participant-profiles","pathname":"/sessions/fixture-33-id%2Fraw/participant-profiles","groups":{"id":"fixture-33-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"read"}],"service":{"kind":"actor"},"auditAllowed":false},"supportedScmProviders":"all","cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"GET /sessions/:id/messages","pathname":"/sessions/fixture-34-id%2Fraw/messages","groups":{"id":"fixture-34-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"read"}],"service":{"kind":"actor"},"auditAllowed":false},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"POST /sessions/:id/pr","pathname":"/sessions/fixture-35-id%2Fraw/pr","groups":{"id":"fixture-35-id%2Fraw"},"authentication":"user-or-service-with-sandbox-fallback","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"collaborate"}],"service":{"kind":"actor"},"auditAllowed":true},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", "{"identity":"POST /sessions/:id/openai-token-refresh","pathname":"/sessions/fixture-36-id%2Fraw/openai-token-refresh","groups":{"id":"fixture-36-id%2Fraw"},"authentication":"sandbox","authorization":{"kind":"none","auditAllowed":false},"supportedScmProviders":"all","cacheControl":null,"hasServiceActorClaims":false}", "{"identity":"POST /sessions/:id/xai-token-refresh","pathname":"/sessions/fixture-37-id%2Fraw/xai-token-refresh","groups":{"id":"fixture-37-id%2Fraw"},"authentication":"sandbox","authorization":{"kind":"none","auditAllowed":false},"supportedScmProviders":"all","cacheControl":null,"hasServiceActorClaims":false}", "{"identity":"POST /sessions/:id/scm-credentials","pathname":"/sessions/fixture-38-id%2Fraw/scm-credentials","groups":{"id":"fixture-38-id%2Fraw"},"authentication":"sandbox","authorization":{"kind":"none","auditAllowed":false},"supportedScmProviders":["github","gitlab"],"cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"GET /sessions/:id/tunnel-urls","pathname":"/sessions/fixture-39-id%2Fraw/tunnel-urls","groups":{"id":"fixture-39-id%2Fraw"},"authentication":"user-or-service-with-sandbox-fallback","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.sandbox_access"}],"auditAllowed":true,"service":{"kind":"actor"}},"supportedScmProviders":"all","cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"PATCH /sessions/:id/title","pathname":"/sessions/fixture-40-id%2Fraw/title","groups":{"id":"fixture-40-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.lifecycle"}],"auditAllowed":true,"service":{"kind":"actor"}},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"POST /sessions/:id/archive","pathname":"/sessions/fixture-41-id%2Fraw/archive","groups":{"id":"fixture-41-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.lifecycle"}],"auditAllowed":true,"service":{"kind":"actor"}},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"POST /sessions/:id/unarchive","pathname":"/sessions/fixture-42-id%2Fraw/unarchive","groups":{"id":"fixture-42-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.lifecycle"}],"auditAllowed":true,"service":{"kind":"actor"}},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"PATCH /sessions/:id/budget","pathname":"/sessions/fixture-43-id%2Fraw/budget","groups":{"id":"fixture-43-id%2Fraw"},"authentication":"user","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.lifecycle"}],"auditAllowed":true,"service":{"kind":"actor"}},"supportedScmProviders":"all","cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"GET /sessions/:id/tunnel-urls","pathname":"/sessions/fixture-39-id%2Fraw/tunnel-urls","groups":{"id":"fixture-39-id%2Fraw"},"authentication":"user-or-service-with-sandbox-fallback","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"sandbox"}],"service":{"kind":"actor"},"auditAllowed":true},"supportedScmProviders":"all","cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"PATCH /sessions/:id/title","pathname":"/sessions/fixture-40-id%2Fraw/title","groups":{"id":"fixture-40-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"lifecycle"}],"service":{"kind":"actor"},"auditAllowed":true},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"POST /sessions/:id/archive","pathname":"/sessions/fixture-41-id%2Fraw/archive","groups":{"id":"fixture-41-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"lifecycle"}],"service":{"kind":"actor"},"auditAllowed":true},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"POST /sessions/:id/unarchive","pathname":"/sessions/fixture-42-id%2Fraw/unarchive","groups":{"id":"fixture-42-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"lifecycle"}],"service":{"kind":"actor"},"auditAllowed":true},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"PATCH /sessions/:id/budget","pathname":"/sessions/fixture-43-id%2Fraw/budget","groups":{"id":"fixture-43-id%2Fraw"},"authentication":"user","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"lifecycle"}],"service":{"kind":"actor"},"auditAllowed":true},"supportedScmProviders":"all","cacheControl":null,"hasServiceActorClaims":false}", "{"identity":"POST /sessions/batch-archive","pathname":"/sessions/batch-archive","groups":{},"authentication":"user","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.bulk_archive"}],"auditAllowed":true,"service":{"kind":"deny"}},"supportedScmProviders":"all","cacheControl":"private, no-store","hasServiceActorClaims":false}", - "{"identity":"POST /sessions/:id/ws-token","pathname":"/sessions/fixture-45-id%2Fraw/ws-token","groups":{"id":"fixture-45-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.read"}],"auditAllowed":false,"service":{"kind":"actor"}},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"POST /sessions/:id/prompt","pathname":"/sessions/fixture-46-id%2Fraw/prompt","groups":{"id":"fixture-46-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.collaborate"}],"auditAllowed":true,"service":{"kind":"actor"}},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"POST /sessions/:id/pull-requests/refresh","pathname":"/sessions/fixture-47-id%2Fraw/pull-requests/refresh","groups":{"id":"fixture-47-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.lifecycle"}],"auditAllowed":true,"service":{"kind":"actor"}},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"POST /sessions/:id/media","pathname":"/sessions/fixture-48-id%2Fraw/media","groups":{"id":"fixture-48-id%2Fraw"},"authentication":"user-or-service-with-sandbox-fallback","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.collaborate"}],"auditAllowed":true,"service":{"kind":"actor"}},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"GET /sessions/:id/media/:artifactId","pathname":"/sessions/fixture-49-id%2Fraw/media/fixture-49-artifactId%2Fraw","groups":{"id":"fixture-49-id%2Fraw","artifactId":"fixture-49-artifactId%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.read"}],"auditAllowed":false,"service":{"kind":"actor","actorlessGrants":[{"service":"slack-bot"}]}},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"POST /sessions/:id/attachments","pathname":"/sessions/fixture-50-id%2Fraw/attachments","groups":{"id":"fixture-50-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.collaborate"}],"auditAllowed":true,"service":{"kind":"actor"}},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"GET /sessions/:id/attachments/:attachmentId","pathname":"/sessions/fixture-51-id%2Fraw/attachments/fixture-51-attachmentId%2Fraw","groups":{"id":"fixture-51-id%2Fraw","attachmentId":"fixture-51-attachmentId%2Fraw"},"authentication":"user-or-service-with-sandbox-fallback","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.read"}],"auditAllowed":false,"service":{"kind":"actor"}},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"GET /sessions/:id/diff","pathname":"/sessions/fixture-52-id%2Fraw/diff","groups":{"id":"fixture-52-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.read"}],"auditAllowed":false,"service":{"kind":"actor"}},"supportedScmProviders":"all","cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"PUT /sessions/:id/diff","pathname":"/sessions/fixture-53-id%2Fraw/diff","groups":{"id":"fixture-53-id%2Fraw"},"authentication":"user-or-service-with-sandbox-fallback","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.collaborate"}],"auditAllowed":true,"service":{"kind":"actor"}},"supportedScmProviders":"all","cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"POST /sessions/:id/diff/failure","pathname":"/sessions/fixture-54-id%2Fraw/diff/failure","groups":{"id":"fixture-54-id%2Fraw"},"authentication":"user-or-service-with-sandbox-fallback","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.collaborate"}],"auditAllowed":true,"service":{"kind":"actor"}},"supportedScmProviders":"all","cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"GET /sessions/:id/diff/:revisionId/files/:fileId","pathname":"/sessions/fixture-55-id%2Fraw/diff/fixture-55-revisionId%2Fraw/files/fixture-55-fileId%2Fraw","groups":{"id":"fixture-55-id%2Fraw","revisionId":"fixture-55-revisionId%2Fraw","fileId":"fixture-55-fileId%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.read"}],"auditAllowed":false,"service":{"kind":"actor"}},"supportedScmProviders":"all","cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"POST /sessions/:id/diff/retry","pathname":"/sessions/fixture-56-id%2Fraw/diff/retry","groups":{"id":"fixture-56-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.lifecycle"}],"auditAllowed":true,"service":{"kind":"actor"}},"supportedScmProviders":"all","cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"GET /sessions/:id/skills","pathname":"/sessions/fixture-57-id%2Fraw/skills","groups":{"id":"fixture-57-id%2Fraw"},"authentication":"user","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.read"}],"auditAllowed":false,"service":{"kind":"actor"}},"supportedScmProviders":"all","cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"POST /sessions/:id/ws-token","pathname":"/sessions/fixture-45-id%2Fraw/ws-token","groups":{"id":"fixture-45-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"read"}],"service":{"kind":"actor"},"auditAllowed":false},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"POST /sessions/:id/prompt","pathname":"/sessions/fixture-46-id%2Fraw/prompt","groups":{"id":"fixture-46-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"collaborate"}],"service":{"kind":"actor"},"auditAllowed":true},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"POST /sessions/:id/pull-requests/refresh","pathname":"/sessions/fixture-47-id%2Fraw/pull-requests/refresh","groups":{"id":"fixture-47-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"lifecycle"}],"service":{"kind":"actor"},"auditAllowed":true},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"POST /sessions/:id/media","pathname":"/sessions/fixture-48-id%2Fraw/media","groups":{"id":"fixture-48-id%2Fraw"},"authentication":"user-or-service-with-sandbox-fallback","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"collaborate"}],"service":{"kind":"actor"},"auditAllowed":true},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"GET /sessions/:id/media/:artifactId","pathname":"/sessions/fixture-49-id%2Fraw/media/fixture-49-artifactId%2Fraw","groups":{"id":"fixture-49-id%2Fraw","artifactId":"fixture-49-artifactId%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"read"}],"service":{"kind":"actor","actorlessGrants":[{"service":"slack-bot"}]},"auditAllowed":false},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"POST /sessions/:id/attachments","pathname":"/sessions/fixture-50-id%2Fraw/attachments","groups":{"id":"fixture-50-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"collaborate"}],"service":{"kind":"actor"},"auditAllowed":true},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"GET /sessions/:id/attachments/:attachmentId","pathname":"/sessions/fixture-51-id%2Fraw/attachments/fixture-51-attachmentId%2Fraw","groups":{"id":"fixture-51-id%2Fraw","attachmentId":"fixture-51-attachmentId%2Fraw"},"authentication":"user-or-service-with-sandbox-fallback","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"read"}],"service":{"kind":"actor"},"auditAllowed":false},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"GET /sessions/:id/diff","pathname":"/sessions/fixture-52-id%2Fraw/diff","groups":{"id":"fixture-52-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"read"}],"service":{"kind":"actor"},"auditAllowed":false},"supportedScmProviders":"all","cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"PUT /sessions/:id/diff","pathname":"/sessions/fixture-53-id%2Fraw/diff","groups":{"id":"fixture-53-id%2Fraw"},"authentication":"user-or-service-with-sandbox-fallback","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"collaborate"}],"service":{"kind":"actor"},"auditAllowed":true},"supportedScmProviders":"all","cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"POST /sessions/:id/diff/failure","pathname":"/sessions/fixture-54-id%2Fraw/diff/failure","groups":{"id":"fixture-54-id%2Fraw"},"authentication":"user-or-service-with-sandbox-fallback","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"collaborate"}],"service":{"kind":"actor"},"auditAllowed":true},"supportedScmProviders":"all","cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"GET /sessions/:id/diff/:revisionId/files/:fileId","pathname":"/sessions/fixture-55-id%2Fraw/diff/fixture-55-revisionId%2Fraw/files/fixture-55-fileId%2Fraw","groups":{"id":"fixture-55-id%2Fraw","revisionId":"fixture-55-revisionId%2Fraw","fileId":"fixture-55-fileId%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"read"}],"service":{"kind":"actor"},"auditAllowed":false},"supportedScmProviders":"all","cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"POST /sessions/:id/diff/retry","pathname":"/sessions/fixture-56-id%2Fraw/diff/retry","groups":{"id":"fixture-56-id%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"lifecycle"}],"service":{"kind":"actor"},"auditAllowed":true},"supportedScmProviders":"all","cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"GET /sessions/:id/skills","pathname":"/sessions/fixture-57-id%2Fraw/skills","groups":{"id":"fixture-57-id%2Fraw"},"authentication":"user","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"read"}],"service":{"kind":"actor"},"auditAllowed":false},"supportedScmProviders":"all","cacheControl":null,"hasServiceActorClaims":false}", "{"identity":"GET /sessions/:id/sandbox-skills","pathname":"/sessions/fixture-58-id%2Fraw/sandbox-skills","groups":{"id":"fixture-58-id%2Fraw"},"authentication":"sandbox","authorization":{"kind":"none","auditAllowed":false},"supportedScmProviders":"all","cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"POST /sessions/:id/children","pathname":"/sessions/fixture-59-id%2Fraw/children","groups":{"id":"fixture-59-id%2Fraw"},"authentication":"user-or-service-with-sandbox-fallback","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.create"},{"kind":"permission","permission":"sessions.collaborate"}],"service":{"kind":"actor"},"auditAllowed":true},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"GET /sessions/:id/children","pathname":"/sessions/fixture-60-id%2Fraw/children","groups":{"id":"fixture-60-id%2Fraw"},"authentication":"user-or-service-with-sandbox-fallback","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.read"}],"auditAllowed":false,"service":{"kind":"actor"}},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"GET /sessions/:id/children/:childId","pathname":"/sessions/fixture-61-id%2Fraw/children/fixture-61-childId%2Fraw","groups":{"id":"fixture-61-id%2Fraw","childId":"fixture-61-childId%2Fraw"},"authentication":"user-or-service-with-sandbox-fallback","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.read"}],"auditAllowed":false,"service":{"kind":"actor"}},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"POST /sessions/:id/children/:childId/cancel","pathname":"/sessions/fixture-62-id%2Fraw/children/fixture-62-childId%2Fraw/cancel","groups":{"id":"fixture-62-id%2Fraw","childId":"fixture-62-childId%2Fraw"},"authentication":"user-or-service-with-sandbox-fallback","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.lifecycle"}],"auditAllowed":true,"service":{"kind":"actor"}},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"POST /sessions/:id/children","pathname":"/sessions/fixture-59-id%2Fraw/children","groups":{"id":"fixture-59-id%2Fraw"},"authentication":"user-or-service-with-sandbox-fallback","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"collaborate"},{"kind":"permission","permission":"sessions.create"}],"service":{"kind":"actor"},"auditAllowed":true},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"GET /sessions/:id/children","pathname":"/sessions/fixture-60-id%2Fraw/children","groups":{"id":"fixture-60-id%2Fraw"},"authentication":"user-or-service-with-sandbox-fallback","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"read"}],"service":{"kind":"actor"},"auditAllowed":false},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"GET /sessions/:id/children/:childId","pathname":"/sessions/fixture-61-id%2Fraw/children/fixture-61-childId%2Fraw","groups":{"id":"fixture-61-id%2Fraw","childId":"fixture-61-childId%2Fraw"},"authentication":"user-or-service-with-sandbox-fallback","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"read"},{"kind":"session","sessionIdParam":"childId","action":"read"}],"service":{"kind":"actor"},"auditAllowed":false},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"POST /sessions/:id/children/:childId/cancel","pathname":"/sessions/fixture-62-id%2Fraw/children/fixture-62-childId%2Fraw/cancel","groups":{"id":"fixture-62-id%2Fraw","childId":"fixture-62-childId%2Fraw"},"authentication":"user-or-service-with-sandbox-fallback","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"read"},{"kind":"session","sessionIdParam":"childId","action":"lifecycle"}],"service":{"kind":"actor"},"auditAllowed":true},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", "{"identity":"POST /sessions/:id/children/:childId/prompt","pathname":"/sessions/fixture-63-id%2Fraw/children/fixture-63-childId%2Fraw/prompt","groups":{"id":"fixture-63-id%2Fraw","childId":"fixture-63-childId%2Fraw"},"authentication":"sandbox","authorization":{"kind":"none","auditAllowed":false},"supportedScmProviders":"all","cacheControl":null,"hasServiceActorClaims":false}", - "{"identity":"POST /sessions/:id/slack-notify","pathname":"/sessions/fixture-64-id%2Fraw/slack-notify","groups":{"id":"fixture-64-id%2Fraw"},"authentication":"user-or-service-with-sandbox-fallback","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"sessions.collaborate"}],"auditAllowed":true,"service":{"kind":"actor"}},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", + "{"identity":"POST /sessions/:id/slack-notify","pathname":"/sessions/fixture-64-id%2Fraw/slack-notify","groups":{"id":"fixture-64-id%2Fraw"},"authentication":"user-or-service-with-sandbox-fallback","authorization":{"kind":"active-user","allOf":[{"kind":"session","sessionIdParam":"id","action":"collaborate"}],"service":{"kind":"actor"},"auditAllowed":true},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", "{"identity":"GET /repos","pathname":"/repos","groups":{},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"repositories.read"}],"auditAllowed":false,"service":{"kind":"actor","actorlessGrants":[{"service":"slack-bot"},{"service":"linear-bot"}]}},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", "{"identity":"PUT /repos/:owner/:name/metadata","pathname":"/repos/fixture-66-owner%2Fraw/fixture-66-name%2Fraw/metadata","groups":{"owner":"fixture-66-owner%2Fraw","name":"fixture-66-name%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"repositories.settings.manage"}],"auditAllowed":true,"service":{"kind":"actor"}},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", "{"identity":"GET /repos/:owner/:name/metadata","pathname":"/repos/fixture-67-owner%2Fraw/fixture-67-name%2Fraw/metadata","groups":{"owner":"fixture-67-owner%2Fraw","name":"fixture-67-name%2Fraw"},"authentication":"user-or-service","authorization":{"kind":"active-user","allOf":[{"kind":"permission","permission":"repositories.read"}],"auditAllowed":false,"service":{"kind":"actor","actorlessGrants":[{"service":"github-bot"}]}},"supportedScmProviders":["github"],"cacheControl":null,"hasServiceActorClaims":false}", diff --git a/packages/control-plane/test/integration/__snapshots__/route-admission-matrix.test.ts.snap b/packages/control-plane/test/integration/__snapshots__/route-admission-matrix.test.ts.snap index 4f0676f4e3..d2b52dabe9 100644 --- a/packages/control-plane/test/integration/__snapshots__/route-admission-matrix.test.ts.snap +++ b/packages/control-plane/test/integration/__snapshots__/route-admission-matrix.test.ts.snap @@ -410,3 +410,101 @@ exports[`route admission matrix > rejects every credentialed route anonymously b "POST /internal/slack-event anonymous=401", ] `; + +exports[`route admission sentinel > admits actorless reads of team and workspace sessions but not private ones 1`] = ` +[ + "actorless-workspace=200", + "actorless-team=200", + "actorless-private=404", +] +`; + +exports[`route admission sentinel > audits one workspace Owner break-glass admission for a private session 1`] = ` +[ + "owner-role-break-glass=200:1", +] +`; + +exports[`route admission sentinel > conceals all team item routes from another team before any handler or DO call 1`] = ` +[ + "PATCH /sessions/:id/read-state other-team=404", + "DELETE /sessions/:id other-team=404", + "GET /sessions/:id/export other-team=404", + "GET /sessions/:id/sandbox-access other-team=404", + "GET /sessions/:id other-team=404", + "POST /sessions/:id/stop other-team=404", + "GET /sessions/:id/events other-team=404", + "GET /sessions/:id/artifacts other-team=404", + "GET /sessions/:id/participants other-team=404", + "GET /sessions/:id/participant-profiles other-team=404", + "GET /sessions/:id/messages other-team=404", + "POST /sessions/:id/pr other-team=404", + "GET /sessions/:id/tunnel-urls other-team=404", + "PATCH /sessions/:id/title other-team=404", + "POST /sessions/:id/archive other-team=404", + "POST /sessions/:id/unarchive other-team=404", + "PATCH /sessions/:id/budget other-team=404", + "POST /sessions/:id/ws-token other-team=404", + "POST /sessions/:id/prompt other-team=404", + "POST /sessions/:id/pull-requests/refresh other-team=404", + "POST /sessions/:id/media other-team=404", + "GET /sessions/:id/media/:artifactId other-team=404", + "POST /sessions/:id/attachments other-team=404", + "GET /sessions/:id/attachments/:attachmentId other-team=404", + "GET /sessions/:id/diff other-team=404", + "PUT /sessions/:id/diff other-team=404", + "POST /sessions/:id/diff/failure other-team=404", + "GET /sessions/:id/diff/:revisionId/files/:fileId other-team=404", + "POST /sessions/:id/diff/retry other-team=404", + "GET /sessions/:id/skills other-team=404", + "POST /sessions/:id/children other-team=404", + "GET /sessions/:id/children other-team=404", + "GET /sessions/:id/children/:childId other-team=404", + "POST /sessions/:id/children/:childId/cancel other-team=404", + "POST /sessions/:id/slack-notify other-team=404", +] +`; + +exports[`route admission sentinel > reports action denials for a same-team Viewer and admits a private collaborator 1`] = ` +[ + "PATCH /sessions/:id/read-state same-team-viewer=200", + "DELETE /sessions/:id same-team-viewer=403", + "GET /sessions/:id/sandbox-access same-team-viewer=403", + "GET /sessions/:id same-team-viewer=200", + "POST /sessions/:id/stop same-team-viewer=403", + "GET /sessions/:id/events same-team-viewer=200", + "GET /sessions/:id/artifacts same-team-viewer=200", + "GET /sessions/:id/participants same-team-viewer=200", + "GET /sessions/:id/participant-profiles same-team-viewer=200", + "GET /sessions/:id/messages same-team-viewer=200", + "POST /sessions/:id/pr same-team-viewer=403", + "GET /sessions/:id/tunnel-urls same-team-viewer=403", + "PATCH /sessions/:id/title same-team-viewer=403", + "POST /sessions/:id/archive same-team-viewer=403", + "POST /sessions/:id/unarchive same-team-viewer=403", + "PATCH /sessions/:id/budget same-team-viewer=403", + "POST /sessions/:id/ws-token same-team-viewer=200", + "POST /sessions/:id/prompt same-team-viewer=403", + "POST /sessions/:id/pull-requests/refresh same-team-viewer=403", + "POST /sessions/:id/media same-team-viewer=403", + "GET /sessions/:id/media/:artifactId same-team-viewer=200", + "POST /sessions/:id/attachments same-team-viewer=403", + "GET /sessions/:id/attachments/:attachmentId same-team-viewer=200", + "GET /sessions/:id/diff same-team-viewer=200", + "PUT /sessions/:id/diff same-team-viewer=403", + "POST /sessions/:id/diff/failure same-team-viewer=403", + "GET /sessions/:id/diff/:revisionId/files/:fileId same-team-viewer=200", + "POST /sessions/:id/diff/retry same-team-viewer=403", + "GET /sessions/:id/skills same-team-viewer=200", + "GET /sessions/:id/children same-team-viewer=200", + "GET /sessions/:id/children/:childId same-team-viewer=200", + "POST /sessions/:id/children/:childId/cancel same-team-viewer=403", + "POST /sessions/:id/slack-notify same-team-viewer=403", +] +`; + +exports[`route admission sentinel > reports action denials for a same-team Viewer and admits a private collaborator 2`] = ` +[ + "collaborator-on-private=200", +] +`; diff --git a/packages/control-plane/test/integration/helpers.ts b/packages/control-plane/test/integration/helpers.ts index bf06956cfa..7c78066828 100644 --- a/packages/control-plane/test/integration/helpers.ts +++ b/packages/control-plane/test/integration/helpers.ts @@ -57,7 +57,7 @@ export const INTEGRATION_WEBSOCKET_TIMEOUT_MS = 2000; const TEST_BROWSER_USER_ID = "11111111111111111111111111111111"; const TEST_BROWSER_ACCOUNT_ID = "test-browser-account"; const TEST_BROWSER_PROVIDER_SUBJECT = "583231"; -type InitialUserRole = Exclude; +type InitialUserRole = BuiltInRoleKey; const DEFAULT_INITIAL_USER_ROLE = "owner" as const; const TEST_BROWSER_SESSION_ID = "test-browser-session"; const TEST_BROWSER_SESSION_TOKEN = "test-browser-session-token"; @@ -96,15 +96,21 @@ async function signCookieValue(value: string, secret: string): Promise { * web request must carry the same compound credential as production. Direct * service-auth tests intentionally build their own bare sig1 requests. */ -async function testBrowserSessionCookie(initialRole: InitialUserRole): Promise { +async function testBrowserSessionCookie( + initialRole: InitialUserRole, + as?: { userId: string; role: BuiltInRoleKey } +): Promise { const secret = env.BROWSER_AUTH_SECRET; if (!secret) throw new Error("BROWSER_AUTH_SECRET is not configured for integration tests"); const now = new Date(); const expiresAt = new Date(now.getTime() + 7 * 24 * 60 * 60 * 1000); const applicationTimestamp = now.getTime(); + const browserUserId = as?.userId ?? TEST_BROWSER_USER_ID; + const sessionId = as ? `test-browser-session-${browserUserId}` : TEST_BROWSER_SESSION_ID; + const sessionToken = as ? `test-browser-token-${browserUserId}` : TEST_BROWSER_SESSION_TOKEN; const existingUser = await env.DB.prepare("SELECT 1 FROM users WHERE id = ?") - .bind(TEST_BROWSER_USER_ID) + .bind(browserUserId) .first(); await env.DB.batch([ env.DB.prepare( @@ -112,11 +118,11 @@ async function testBrowserSessionCookie(initialRole: InitialUserRole): Promise { ), ]); + await env.DB.prepare( + `INSERT INTO sessions (id, repo_owner, repo_name, status, user_id, created_at, updated_at) + VALUES ('parent', 'acme', 'app', 'completed', ?, 1, 1)` + ) + .bind(user!.id) + .run(); + const response = await serviceFetch("https://cp.test/sessions/parent/children", { method: "POST", headers: { "Content-Type": "application/json" }, diff --git a/packages/control-plane/test/integration/route-admission-matrix.test.ts b/packages/control-plane/test/integration/route-admission-matrix.test.ts index 570b897c26..223fb17dfe 100644 --- a/packages/control-plane/test/integration/route-admission-matrix.test.ts +++ b/packages/control-plane/test/integration/route-admission-matrix.test.ts @@ -9,7 +9,7 @@ */ import { SELF, env } from "cloudflare:test"; -import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { buildServiceAuthHeaders } from "@open-inspect/shared/service-auth"; import { createExecutionContext } from "cloudflare:test"; import { cloudflareHost, createControlPlaneHttpHandler } from "../../src/cloudflare/http-host"; @@ -18,6 +18,8 @@ import { listRouteContracts, type RouteContract } from "../../src/routing/route- import { createCloudflareEnv } from "../../src/cloudflare/platform"; import { AutomationStore, type AutomationRow } from "../../src/db/automation-store"; import { TeamStore } from "../../src/db/teams"; +import { TeamMembershipStore } from "../../src/db/team-memberships"; +import { SessionCollaboratorStore } from "../../src/db/session-collaborators"; import { catalog } from "../../src/routes/catalog"; import { Hono } from "hono"; import { admit } from "../../src/routing/admit"; @@ -412,6 +414,11 @@ describe("route admission sentinel", { timeout: MATRIX_TIMEOUT_MS }, () => { ); } const handle = createControlPlaneHttpHandler([shadow]); + let teamSessionId = ""; + let privateSessionId = ""; + const OTHER_MEMBER = "33333333333333333333333333333333"; + const TEAM_VIEWER = "44444444444444444444444444444444"; + const COLLABORATOR = "55555555555555555555555555555555"; beforeAll(async () => { await cleanD1Tables(); @@ -428,6 +435,25 @@ describe("route admission sentinel", { timeout: MATRIX_TIMEOUT_MS }, () => { joinPolicy: "open", }) ).id; + for (const [userId, role] of [ + [OTHER_MEMBER, "member"], + [TEAM_VIEWER, "viewer"], + [COLLABORATOR, "member"], + ] as const) { + await serviceRequestHeaders(`${BASE}/me/authorization`, { as: { userId, role } }); + } + await new TeamMembershipStore(env.DB).add(fixtures.teamId, TEAM_VIEWER); + teamSessionId = await createReadySession(); + privateSessionId = await createReadySession(); + await env.DB.batch([ + env.DB.prepare( + "UPDATE sessions SET owner_team_id = ?, visibility = 'team' WHERE id = ?" + ).bind(fixtures.teamId, teamSessionId), + env.DB.prepare( + "UPDATE sessions SET owner_team_id = ?, visibility = 'private', user_id = ? WHERE id = ?" + ).bind(fixtures.teamId, OTHER_MEMBER, privateSessionId), + ]); + await new SessionCollaboratorStore(env.DB).add(privateSessionId, COLLABORATOR, OTHER_MEMBER); }, MATRIX_TIMEOUT_MS); afterAll(async () => { @@ -477,6 +503,7 @@ describe("route admission sentinel", { timeout: MATRIX_TIMEOUT_MS }, () => { : isAutomationRoute(route) ? fixtures.automationId : sessionId, + childId: fixtures.sandboxSessionId, })}`; const method = route.method; const expectReach = async ( @@ -548,6 +575,7 @@ describe("route admission sentinel", { timeout: MATRIX_TIMEOUT_MS }, () => { }); it("delivers path segments to handlers decoded exactly once", async () => { + await initSession({ sessionName: "abc/def", userId: BROWSER_USER_ID }); // Every production contract, admitted by its own policy, in front of a // handler that echoes the parameters Hono decoded. const echo = new Hono(); @@ -586,4 +614,131 @@ describe("route admission sentinel", { timeout: MATRIX_TIMEOUT_MS }, () => { await expect(response.json(), url).resolves.toEqual({ groups }); } }); + + it("conceals all team item routes from another team before any handler or DO call", async () => { + const get = vi.fn(() => { + throw new Error("Denied route reached the Durable Object"); + }); + const requestEnv = createCloudflareEnv({ + ...env, + TEAMS_ENFORCEMENT: "on", + SESSION: new Proxy(env.SESSION, { + get(target, property, receiver) { + if (property === "get") return get; + return Reflect.get(target, property, receiver); + }, + }), + }); + const observed: string[] = []; + for (const route of routes.filter( + (item) => isSessionRoute(item) && item.authorization.kind === "active-user" + )) { + const identity = `${route.method} ${route.path}`; + const url = `${BASE}${materialize(route, { id: teamSessionId, childId: fixtures.sandboxSessionId })}`; + const headers = await serviceRequestHeaders(url, { + method: route.method, + as: { userId: OTHER_MEMBER, role: "member" }, + }); + const response = await handle( + new Request(url, { method: route.method, headers }), + requestEnv, + createExecutionContext() + ); + observed.push(`${identity} other-team=${response.status}`); + expect(response.status, identity).toBe(404); + await expect(response.json(), identity).resolves.toEqual({ error: "Session not found" }); + } + expect(get).not.toHaveBeenCalled(); + expect(observed).toMatchSnapshot(); + }); + + it("reports action denials for a same-team Viewer and admits a private collaborator", async () => { + const observed: string[] = []; + for (const route of routes.filter( + (item) => isSessionRoute(item) && item.authorization.kind === "active-user" + )) { + if ( + route.path.endsWith("/export") || + (route.path.endsWith("/children") && route.method === "POST") + ) + continue; + const url = `${BASE}${materialize(route, { id: teamSessionId, childId: fixtures.sandboxSessionId })}`; + const headers = await serviceRequestHeaders(url, { + method: route.method, + as: { userId: TEAM_VIEWER, role: "viewer" }, + }); + const response = await handle( + new Request(url, { method: route.method, headers }), + createCloudflareEnv({ ...env, TEAMS_ENFORCEMENT: "on" }), + createExecutionContext() + ); + const expected = + route.authorization.kind === "active-user" && + route.authorization.allOf.every( + (entry) => entry.kind !== "session" || entry.action === "read" + ) + ? 200 + : 403; + observed.push(`${route.method} ${route.path} same-team-viewer=${response.status}`); + expect(response.status, `${route.method} ${route.path}`).toBe(expected); + if (expected === 403) + await expect(response.json()).resolves.toMatchObject({ reason_code: "missing_permission" }); + } + expect(observed).toMatchSnapshot(); + const url = `${BASE}/sessions/${privateSessionId}/events`; + const headers = await serviceRequestHeaders(url, { + as: { userId: COLLABORATOR, role: "member" }, + }); + const response = await handle( + new Request(url, { headers }), + createCloudflareEnv({ ...env, TEAMS_ENFORCEMENT: "on" }), + createExecutionContext() + ); + expect(response.status).toBe(200); + expect([`collaborator-on-private=${response.status}`]).toMatchSnapshot(); + }); + + it("admits actorless reads of team and workspace sessions but not private ones", async () => { + const observed: string[] = []; + for (const id of [fixtures.readonlySessionId, teamSessionId, privateSessionId]) { + const url = `${BASE}/sessions/${id}/events`; + const response = await handle( + new Request(url, { + headers: await botHeaders(url, "GET", "slack-bot"), + }), + createCloudflareEnv({ ...env, TEAMS_ENFORCEMENT: "on" }), + createExecutionContext() + ); + observed.push( + `actorless-${id === fixtures.readonlySessionId ? "workspace" : id === teamSessionId ? "team" : "private"}=${response.status}` + ); + expect(response.status).toBe(id === privateSessionId ? 404 : 200); + } + expect(observed).toMatchSnapshot(); + }); + + it("audits one workspace Owner break-glass admission for a private session", async () => { + const url = `${BASE}/sessions/${privateSessionId}`; + const headers = await serviceRequestHeaders(url); + const response = await handle( + new Request(url, { headers }), + createCloudflareEnv({ ...env, TEAMS_ENFORCEMENT: "on" }), + createExecutionContext() + ); + expect(response.status).toBe(200); + const audits = await env.DB.prepare( + "SELECT resource_type, resource_id, team_id, actor_user_id_snapshot FROM authorization_audit_events WHERE action = 'session.private_break_glass'" + ).all(); + expect(audits.results).toEqual([ + { + resource_type: "session", + resource_id: privateSessionId, + team_id: fixtures.teamId, + actor_user_id_snapshot: BROWSER_USER_ID, + }, + ]); + expect([ + `owner-role-break-glass=${response.status}:${audits.results.length}`, + ]).toMatchSnapshot(); + }); }); diff --git a/packages/control-plane/test/integration/session-access-routes.test.ts b/packages/control-plane/test/integration/session-access-routes.test.ts new file mode 100644 index 0000000000..3465bc861a --- /dev/null +++ b/packages/control-plane/test/integration/session-access-routes.test.ts @@ -0,0 +1,473 @@ +import { createExecutionContext, env } from "cloudflare:test"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { TeamStore } from "../../src/db/teams"; +import { TeamMembershipStore } from "../../src/db/team-memberships"; +import { SessionCollaboratorStore } from "../../src/db/session-collaborators"; +import { cleanD1Tables } from "./cleanup"; +import { + initSession, + routeRequest, + seedActiveUser, + serviceRequestHeaders, + waitForSandboxStatus, +} from "./helpers"; + +const BASE = "https://test.local"; +const OWNER = "11111111111111111111111111111111"; +const MEMBER = "22222222222222222222222222222222"; +const CREATOR = "33333333333333333333333333333333"; + +async function fetchMode( + path: string, + mode: string, + options: { + method?: string; + as?: { userId: string; role: "owner" | "member" | "viewer" }; + body?: string; + service?: "linear-bot"; + } = {} +) { + const url = `${BASE}${path}`; + const method = options.method ?? "GET"; + return routeRequest( + new Request(url, { + method, + headers: await serviceRequestHeaders(url, { + method, + body: options.body, + as: options.as, + service: options.service, + }), + body: options.body, + }), + { ...env, TEAMS_ENFORCEMENT: mode }, + createExecutionContext() + ); +} + +async function auditRows(action: string) { + return ( + await env.DB.prepare( + "SELECT action, resource_type, resource_id, team_id, reason_code, actor_user_id_snapshot FROM authorization_audit_events WHERE action = ? ORDER BY occurred_at" + ) + .bind(action) + .all() + ).results; +} + +describe("HTTP session access by enforcement mode", () => { + beforeEach(async () => { + await cleanD1Tables(); + expect((await fetchMode("/me/authorization", "on")).status).toBe(200); + expect( + ( + await fetchMode("/me/authorization", "on", { + as: { userId: MEMBER, role: "member" }, + }) + ).status + ).toBe(200); + await seedActiveUser(CREATOR); + }); + + async function session(visibility: "team" | "private") { + const team = await new TeamStore(env.DB).create({ + slug: `access-${crypto.randomUUID()}`, + name: "Access Team", + joinPolicy: "invite_only", + }); + const { sessionName, stub } = await initSession({ userId: CREATOR }); + await waitForSandboxStatus(stub, "failed"); + await env.DB.prepare("UPDATE sessions SET owner_team_id = ?, visibility = ? WHERE id = ?") + .bind(team.id, visibility, sessionName) + .run(); + return { sessionName, team }; + } + + it("conceals a team session on read and token mint when enforcement is on", async () => { + const { sessionName, team } = await session("team"); + const as = { userId: MEMBER, role: "member" } as const; + const snapshot = await fetchMode(`/sessions/${sessionName}`, "on", { as }); + const token = await fetchMode(`/sessions/${sessionName}/ws-token`, "on", { + as, + method: "POST", + }); + expect(snapshot.status).toBe(404); + expect(await snapshot.json()).toEqual({ error: "Session not found" }); + expect(token.status).toBe(404); + const denied = await auditRows("authorization.request_denied"); + expect(denied.filter((row) => row.reason_code === "session_not_visible")).toHaveLength(2); + expect(denied.find((row) => row.reason_code === "session_not_visible")?.team_id).toBe(team.id); + }); + + it("defers the team and delete rules in shadow but records each would-be denial", async () => { + const { sessionName, team } = await session("team"); + const as = { userId: MEMBER, role: "member" } as const; + const snapshot = await fetchMode(`/sessions/${sessionName}`, "shadow", { as }); + expect(snapshot.status).toBe(200); + const shadowRows = (await auditRows("authorization.request_allowed")).filter( + (row) => typeof row.reason_code === "string" && row.reason_code.startsWith("shadow_denied:") + ); + expect(shadowRows).toMatchObject([ + { reason_code: "shadow_denied:not_member", team_id: team.id }, + ]); + + await new TeamMembershipStore(env.DB).add(team.id, MEMBER); + const denied = await fetchMode(`/sessions/${sessionName}`, "on", { as }); + expect(denied.status).toBe(200); + const deletion = await fetchMode(`/sessions/${sessionName}`, "on", { as, method: "DELETE" }); + expect(deletion.status).toBe(403); + expect(await deletion.json()).toEqual({ + error: "Forbidden", + code: "session_action_denied", + reason_code: "not_owner_or_lead", + }); + await new TeamMembershipStore(env.DB).setRole(team.id, MEMBER, "lead"); + expect( + (await fetchMode(`/sessions/${sessionName}`, "on", { as, method: "DELETE" })).status + ).toBe(200); + }); + + it("writes one shadow audit row with the real status for reads and mutations", async () => { + const { sessionName } = await session("team"); + for (const [path, method, body] of [ + [`/sessions/${sessionName}`, "GET", undefined], + [`/sessions/${sessionName}/budget`, "PATCH", JSON.stringify({ maxCostUsd: 20 })], + ] as const) { + const response = await fetchMode(path, "shadow", { + method, + body, + as: { userId: MEMBER, role: "member" }, + }); + const rows = ( + await env.DB.prepare( + "SELECT reason_code, metadata_json FROM authorization_audit_events WHERE request_id = ? AND action = 'authorization.request_allowed'" + ) + .bind(response.headers.get("x-request-id")) + .all() + ).results; + expect(response.status).toBe(method === "GET" ? 200 : 403); + expect(rows).toHaveLength(1); + expect(rows[0].reason_code).toBe("shadow_denied:not_member"); + expect(JSON.parse(String(rows[0].metadata_json))).toMatchObject({ + httpStatus: response.status, + responseCode: "shadow_denied:not_member", + }); + } + }); + + it("records batch shadow denials in its one response-time audit row", async () => { + const { sessionName } = await session("team"); + await env.DB.batch([ + env.DB.prepare( + "INSERT INTO roles (id, key, name, normalized_name, is_system) VALUES ('role_batch_shadow', NULL, 'Batch Shadow', 'batch shadow', 0)" + ), + env.DB.prepare( + `INSERT INTO role_permissions (role_id, permission_id) + VALUES ('role_batch_shadow', 'sessions.bulk_archive'), + ('role_batch_shadow', 'sessions.read'), + ('role_batch_shadow', 'sessions.lifecycle')` + ), + env.DB.prepare( + "UPDATE user_role_assignments SET role_id = 'role_batch_shadow' WHERE user_id = ?" + ).bind(MEMBER), + ]); + const response = await fetchMode("/sessions/batch-archive", "shadow", { + method: "POST", + as: { userId: MEMBER, role: "member" }, + body: JSON.stringify({ sessionIds: [sessionName] }), + }); + const rows = ( + await env.DB.prepare( + "SELECT reason_code, metadata_json FROM authorization_audit_events WHERE request_id = ? AND action = 'authorization.request_allowed'" + ) + .bind(response.headers.get("x-request-id")) + .all() + ).results; + expect(rows).toHaveLength(1); + expect(rows[0].reason_code).toBe("shadow_denied:batch"); + expect(JSON.parse(String(rows[0].metadata_json))).toMatchObject({ + httpStatus: response.status, + responseCode: "shadow_denied:batch", + shadowDenials: [{ sessionId: sessionName, reason: "not_member" }], + }); + }); + + it("allows legacy deletion in shadow and off while shadow audits the ownership denial", async () => { + const as = { userId: MEMBER, role: "member" } as const; + const shadow = await session("team"); + await new TeamMembershipStore(env.DB).add(shadow.team.id, MEMBER); + expect( + (await fetchMode(`/sessions/${shadow.sessionName}`, "shadow", { method: "DELETE", as })) + .status + ).toBe(200); + expect( + (await auditRows("authorization.request_allowed")).filter( + (row) => row.reason_code === "shadow_denied:not_owner_or_lead" + ) + ).toHaveLength(1); + const off = await session("team"); + expect( + (await fetchMode(`/sessions/${off.sessionName}`, "off", { method: "DELETE", as })).status + ).toBe(200); + }); + + it("conceals another team's export even when the viewer holds sessions.export", async () => { + const { sessionName } = await session("team"); + await env.DB.batch([ + env.DB.prepare( + `INSERT INTO roles (id, key, name, normalized_name, is_system) + VALUES ('role_export_reader', NULL, 'Export Reader', 'export reader', 0)` + ), + env.DB.prepare( + `INSERT INTO role_permissions (role_id, permission_id) + VALUES ('role_export_reader', 'sessions.read'), + ('role_export_reader', 'sessions.export')` + ), + env.DB.prepare( + "UPDATE user_role_assignments SET role_id = 'role_export_reader' WHERE user_id = ?" + ).bind(MEMBER), + ]); + const response = await fetchMode(`/sessions/${sessionName}/export`, "on", { + as: { userId: MEMBER, role: "member" }, + }); + expect(response.status).toBe(404); + expect(await response.json()).toEqual({ error: "Session not found" }); + }); + + it("keeps private sessions concealed in all modes and audits Owner break-glass once per read", async () => { + const { sessionName, team } = await session("private"); + const as = { userId: MEMBER, role: "member" } as const; + await new TeamMembershipStore(env.DB).add(team.id, MEMBER); + for (const mode of ["off", "shadow", "on"] as const) { + expect((await fetchMode(`/sessions/${sessionName}`, mode, { as })).status).toBe(404); + } + expect((await fetchMode(`/sessions/${sessionName}`, "on")).status).toBe(200); + expect(await auditRows("session.private_break_glass")).toMatchObject([ + { + resource_type: "session", + resource_id: sessionName, + team_id: team.id, + actor_user_id_snapshot: OWNER, + }, + ]); + await new SessionCollaboratorStore(env.DB).add(sessionName, MEMBER, OWNER); + expect((await fetchMode(`/sessions/${sessionName}`, "on", { as })).status).toBe(200); + expect(await auditRows("session.private_break_glass")).toHaveLength(1); + }); + + it("does not query memberships in off mode", async () => { + const { sessionName } = await session("team"); + const list = vi.spyOn(TeamMembershipStore.prototype, "listForUser"); + try { + expect( + ( + await fetchMode(`/sessions/${sessionName}`, "off", { + as: { userId: MEMBER, role: "member" }, + }) + ).status + ).toBe(200); + expect(list).not.toHaveBeenCalled(); + } finally { + list.mockRestore(); + } + }); + + it("loads collaborators but not memberships for an off-mode private session", async () => { + const { sessionName } = await session("private"); + const members = vi.spyOn(TeamMembershipStore.prototype, "listForUser"); + const collaborators = vi.spyOn(SessionCollaboratorStore.prototype, "listUserIds"); + try { + expect((await fetchMode(`/sessions/${sessionName}`, "off")).status).toBe(200); + expect(members).not.toHaveBeenCalled(); + expect(collaborators).toHaveBeenCalledOnce(); + } finally { + members.mockRestore(); + collaborators.mockRestore(); + } + }); + + it("admits actorless Linear stop on non-private sessions in every mode, but hides private sessions", async () => { + const team = await session("team"); + const privateSession = await session("private"); + const workspace = await initSession({ userId: CREATOR }); + await waitForSandboxStatus(workspace.stub, "failed"); + for (const mode of ["off", "shadow", "on"] as const) { + for (const id of [team.sessionName, workspace.sessionName]) { + const response = await fetchMode(`/sessions/${id}/stop`, mode, { + method: "POST", + service: "linear-bot", + }); + expect(response.status).toBe(200); + } + expect( + ( + await fetchMode(`/sessions/${privateSession.sessionName}/stop`, mode, { + method: "POST", + service: "linear-bot", + }) + ).status + ).toBe(404); + } + expect( + (await auditRows("authorization.request_allowed")).filter( + (row) => row.reason_code === "shadow_denied:missing_permission" + ) + ).toEqual([]); + }); + + it("conceals an invisible child even when its parent is visible", async () => { + const parent = await initSession({ userId: CREATOR }); + await waitForSandboxStatus(parent.stub, "failed"); + const child = await session("team"); + await env.DB.prepare("UPDATE sessions SET parent_session_id = ? WHERE id = ?") + .bind(parent.sessionName, child.sessionName) + .run(); + const as = { userId: MEMBER, role: "member" } as const; + expect( + ( + await fetchMode(`/sessions/${parent.sessionName}/children/${child.sessionName}`, "on", { + as, + }) + ).status + ).toBe(404); + expect( + ( + await fetchMode( + `/sessions/${parent.sessionName}/children/${child.sessionName}/cancel`, + "on", + { as, method: "POST" } + ) + ).status + ).toBe(404); + expect( + (await auditRows("authorization.request_denied")).filter( + (row) => row.reason_code === "session_not_visible" + ) + ).toMatchObject([{ team_id: child.team.id }, { team_id: child.team.id }]); + }); + + it("audits both private reads when an Owner accesses a private child", async () => { + const parent = await session("private"); + const child = await session("private"); + await env.DB.prepare("UPDATE sessions SET parent_session_id = ? WHERE id = ?") + .bind(parent.sessionName, child.sessionName) + .run(); + await fetchMode(`/sessions/${parent.sessionName}/children/${child.sessionName}`, "on"); + expect( + (await auditRows("session.private_break_glass")).map((row) => row.resource_id).sort() + ).toEqual([parent.sessionName, child.sessionName].sort()); + }); + + it("audits a permitted private read before a later handler denial", async () => { + const { sessionName } = await session("private"); + const response = await fetchMode(`/sessions/${sessionName}/budget`, "off", { + method: "PATCH", + body: JSON.stringify({ maxCostUsd: 20 }), + }); + expect(response.status).toBe(403); + expect((await auditRows("session.private_break_glass")).map((row) => row.resource_id)).toEqual([ + sessionName, + ]); + }); + + it("responds 503 to an invalid mode for both item and batch routes", async () => { + expect((await fetchMode("/sessions/missing", "invalid")).status).toBe(503); + expect( + ( + await fetchMode("/sessions/batch-archive", "invalid", { + method: "POST", + body: JSON.stringify({ sessionIds: ["missing"] }), + }) + ).status + ).toBe(503); + }); + + it("skips hidden and action-denied batch targets independently", async () => { + const hidden = await session("private"); + const visible = await session("team"); + await new TeamMembershipStore(env.DB).add(visible.team.id, MEMBER); + await env.DB.batch([ + env.DB.prepare( + `INSERT INTO roles (id, key, name, normalized_name, is_system) + VALUES ('role_batch_viewer', NULL, 'Batch Viewer', 'batch viewer', 0)` + ), + env.DB.prepare( + `INSERT INTO role_permissions (role_id, permission_id) + VALUES ('role_batch_viewer', 'sessions.bulk_archive'), + ('role_batch_viewer', 'sessions.read')` + ), + env.DB.prepare( + "UPDATE user_role_assignments SET role_id = 'role_batch_viewer' WHERE user_id = ?" + ).bind(MEMBER), + ]); + const memberships = vi.spyOn(TeamMembershipStore.prototype, "listForUser"); + const response = await fetchMode("/sessions/batch-archive", "on", { + method: "POST", + as: { userId: MEMBER, role: "member" }, + body: JSON.stringify({ sessionIds: [hidden.sessionName, visible.sessionName] }), + }); + expect(memberships).toHaveBeenCalledOnce(); + memberships.mockRestore(); + expect(response.status).toBe(200); + expect(await response.json()).toEqual({ + results: [], + skipped: [ + { sessionId: hidden.sessionName, reason: "not_found" }, + { sessionId: visible.sessionName, reason: "missing_permission" }, + ], + }); + }); + + it.each(["off", "shadow", "on"] as const)( + "preserves bulk-only custom-role archiving in %s mode", + async (mode) => { + const { sessionName, team } = await session("team"); + await new TeamMembershipStore(env.DB).add(team.id, MEMBER); + await env.DB.batch([ + env.DB.prepare( + `INSERT INTO roles (id, key, name, normalized_name, is_system) + VALUES ('role_bulk_only', NULL, 'Bulk Only', 'bulk only', 0)` + ), + env.DB.prepare( + `INSERT INTO role_permissions (role_id, permission_id) + VALUES ('role_bulk_only', 'sessions.bulk_archive'), + ('role_bulk_only', 'sessions.read')` + ), + env.DB.prepare( + "UPDATE user_role_assignments SET role_id = 'role_bulk_only' WHERE user_id = ?" + ).bind(MEMBER), + ]); + + const response = await fetchMode("/sessions/batch-archive", mode, { + method: "POST", + as: { userId: MEMBER, role: "member" }, + body: JSON.stringify({ sessionIds: [sessionName] }), + }); + expect(response.status).toBe(200); + expect(await response.json()).toEqual( + mode === "on" + ? { results: [], skipped: [{ sessionId: sessionName, reason: "missing_permission" }] } + : { results: [{ sessionId: sessionName, outcome: "archived" }], skipped: [] } + ); + if (mode === "shadow") { + expect( + (await auditRows("authorization.request_allowed")).some( + (row) => row.reason_code === "shadow_denied:batch" + ) + ).toBe(true); + } + } + ); + + it("lists, idempotently adds, and removes collaborators", async () => { + const { sessionName } = await session("private"); + const store = new SessionCollaboratorStore(env.DB); + expect(await store.add(sessionName, MEMBER, OWNER)).toBe(true); + expect(await store.add(sessionName, MEMBER, OWNER)).toBe(false); + expect(await store.listUserIds(sessionName)).toEqual([MEMBER]); + expect(await store.listForUser(MEMBER)).toEqual([sessionName]); + expect(await store.remove(sessionName, MEMBER)).toBe(true); + expect(await store.remove(sessionName, MEMBER)).toBe(false); + expect(await store.listUserIds(sessionName)).toEqual([]); + }); +}); diff --git a/packages/control-plane/test/integration/session-batch-archive.test.ts b/packages/control-plane/test/integration/session-batch-archive.test.ts index 946f2a9a0d..c2ab14ac79 100644 --- a/packages/control-plane/test/integration/session-batch-archive.test.ts +++ b/packages/control-plane/test/integration/session-batch-archive.test.ts @@ -46,7 +46,8 @@ describe("session batch archive", () => { const allowed = await member(); expect(allowed.status).toBe(200); expect(await allowed.json()).toEqual({ - results: [{ sessionId: "missing", outcome: "not_found" }], + results: [], + skipped: [{ sessionId: "missing", reason: "not_found" }], }); await env.DB.prepare("UPDATE user_role_assignments SET role_id = ? WHERE user_id = ?") .bind("role_builtin_member", USER_ID) @@ -91,13 +92,14 @@ describe("session batch archive", () => { expect(sessionBatchArchiveResponseSchema.parse(await response.json())).toEqual({ results: [ { sessionId: ready.sessionName, outcome: "archived" }, - { sessionId: "missing", outcome: "not_found" }, { sessionId: cancelled.sessionName, outcome: "skipped_cancelled" }, { sessionId: queued.sessionName, outcome: "skipped_queued_work" }, ], + skipped: [{ sessionId: "missing", reason: "not_found" }], }); expect(await (await post([ready.sessionName])).json()).toEqual({ results: [{ sessionId: ready.sessionName, outcome: "already_archived" }], + skipped: [], }); }); @@ -120,6 +122,7 @@ describe("session batch archive", () => { expect(response.status).toBe(200); expect(await response.json()).toEqual({ results: [{ sessionId: sessionName, outcome: "already_archived" }], + skipped: [], }); expect( await env.DB.prepare("SELECT status, updated_at FROM sessions WHERE id = ?") @@ -134,7 +137,8 @@ describe("session batch archive", () => { await waitForSandboxStatus(stub, "failed"); await env.DB.prepare("DELETE FROM sessions WHERE id = ?").bind(sessionName).run(); expect(await (await post([sessionName])).json()).toEqual({ - results: [{ sessionId: sessionName, outcome: "failed" }], + results: [], + skipped: [{ sessionId: sessionName, reason: "not_found" }], }); }); diff --git a/packages/shared/src/types/audit-events.test.ts b/packages/shared/src/types/audit-events.test.ts index aaaaad602f..28bc73a5c1 100644 --- a/packages/shared/src/types/audit-events.test.ts +++ b/packages/shared/src/types/audit-events.test.ts @@ -124,6 +124,7 @@ describe("interpretAuditEvent", () => { }); it.each([ + "session.private_break_glass", "team.created", "team.updated", "team.archived", diff --git a/packages/shared/src/types/audit-events.ts b/packages/shared/src/types/audit-events.ts index d6bb2ad320..7d55b25622 100644 --- a/packages/shared/src/types/audit-events.ts +++ b/packages/shared/src/types/audit-events.ts @@ -71,6 +71,7 @@ export const AUTHORIZATION_DECISION_ACTIONS = { /** Actions written by the operation owner alongside the change; their result is the domain outcome. */ export const AUDIT_OPERATION_ACTIONS = [ + "session.private_break_glass", "workspace.member_role_updated", "workspace.member_status_updated", "workspace.default_role_assigned", diff --git a/packages/shared/src/types/session-archive.ts b/packages/shared/src/types/session-archive.ts index c2de608030..a368126757 100644 --- a/packages/shared/src/types/session-archive.ts +++ b/packages/shared/src/types/session-archive.ts @@ -42,6 +42,12 @@ export type SessionBatchArchiveResult = z.infer; diff --git a/packages/web/src/components/settings/audit-log-settings.test.tsx b/packages/web/src/components/settings/audit-log-settings.test.tsx index 852f699a2e..875725087d 100644 --- a/packages/web/src/components/settings/audit-log-settings.test.tsx +++ b/packages/web/src/components/settings/audit-log-settings.test.tsx @@ -110,6 +110,12 @@ describe("AuditLogSettings", () => { expect(article.getByText("Applied")).toBeInTheDocument(); }); + it("labels private session break-glass reads as operations", () => { + const article = renderSingle(createEvent("applied", { action: "session.private_break_glass" })); + expect(article.getByText("Private session break-glass read")).toBeInTheDocument(); + expect(article.getByText("Applied")).toBeInTheDocument(); + }); + it("renders outcomes, stable summaries, timestamps, and expandable structured details", async () => { hook.events = [ createEvent("applied"), diff --git a/packages/web/src/components/settings/audit-log-settings.tsx b/packages/web/src/components/settings/audit-log-settings.tsx index b24b3e9c3d..372f6c07a5 100644 --- a/packages/web/src/components/settings/audit-log-settings.tsx +++ b/packages/web/src/components/settings/audit-log-settings.tsx @@ -40,6 +40,7 @@ const UNRECOGNIZED: BadgeTreatment = { }; const OPERATION_LABELS: Record = { + "session.private_break_glass": "Private session break-glass read", "workspace.member_role_updated": "Member role updated", "workspace.member_status_updated": "Member status updated", "workspace.default_role_assigned": "Default role assigned", diff --git a/terraform/environments/aws-production/main.tf b/terraform/environments/aws-production/main.tf index ff198baa84..26868c6205 100644 --- a/terraform/environments/aws-production/main.tf +++ b/terraform/environments/aws-production/main.tf @@ -44,6 +44,7 @@ module "control_plane" { LOG_LEVEL = "info" SANDBOX_PROVIDER = "modal" UNSAFE_ALLOW_ALL_USERS = "false" + TEAMS_ENFORCEMENT = "shadow" }, var.config) tags = { Environment = local.environment } diff --git a/terraform/environments/production/variables.tf b/terraform/environments/production/variables.tf index c66f7a370f..d9d9d6cb5e 100644 --- a/terraform/environments/production/variables.tf +++ b/terraform/environments/production/variables.tf @@ -699,6 +699,17 @@ variable "sandbox_inactivity_timeout_ms" { default = 600000 } +variable "teams_enforcement" { + description = "Session team authorization mode; private visibility applies in every mode." + type = string + default = "shadow" + + validation { + condition = contains(["off", "shadow", "on"], var.teams_enforcement) + error_message = "teams_enforcement must be 'off', 'shadow', or 'on'." + } +} + variable "sandbox_boot_timeout_ms" { description = "Milliseconds a sandbox whose bridge has connected may keep booting (clone, setup.sh, start.sh, agent start) before OpenInspect fails it and the prompt it was for." type = number diff --git a/terraform/environments/production/workers-control-plane.tf b/terraform/environments/production/workers-control-plane.tf index dc90293fe4..3e0178a941 100644 --- a/terraform/environments/production/workers-control-plane.tf +++ b/terraform/environments/production/workers-control-plane.tf @@ -102,6 +102,7 @@ module "control_plane_worker" { SANDBOX_PROVIDER = { value = var.sandbox_provider } SANDBOX_INACTIVITY_TIMEOUT_MS = { value = tostring(var.sandbox_inactivity_timeout_ms) } SANDBOX_BOOT_TIMEOUT_MS = { value = tostring(var.sandbox_boot_timeout_ms) } + TEAMS_ENFORCEMENT = { value = var.teams_enforcement } }, local.github_oauth_enabled ? { GITHUB_CLIENT_ID = { value = trimspace(var.github_client_id) } From 7b49f86a3af73930afcdb96729115843fd741c38 Mon Sep 17 00:00:00 2001 From: Cole Murray Date: Mon, 28 Sep 2026 20:53:39 -0700 Subject: [PATCH 05/13] feat: support Claude Sonnet 5.5 (#2128) ## Summary - Add `anthropic/claude-sonnet-5-5` to the shared model catalog, model picker, and documentation, with low through max adaptive-thinking efforts and a high default. - Upgrade the Claude Agent SDK pin from 0.2.158 to 0.2.161 (bundled Claude Code 2.1.284, which adds Sonnet 5.5), regenerate runtime and Modal locks, and raise only the Claude harness prepared-image floor to generation 74. - Extend Claude harness and OpenCode reasoning-contract coverage for the new model. OpenCode 1.18.29 resolves it via the refreshed model registry; no OpenCode binary change is needed. ## Verification - `npm run build -w @open-inspect/shared` - `npm test -w @open-inspect/shared` (1,052 passed) - `npm run typecheck` and `npm run lint` - `uv run --frozen --project packages/sandbox-runtime --extra dev pytest packages/sandbox-runtime/tests -q` (1,390 passed, 3 skipped) - OpenCode wire contract with `OPENCODE_TEST_BINARY` (3 passed, fake provider, no live Anthropic request) - Modal manifest and sandbox image bundle tests, sandbox image lock check, Ruff and Prettier checks No authenticated Anthropic model call or provider-native image build was run. --- *Created with [Open-Inspect](https://open-inspect-prod.vercel.app/session/aabcae601ce9b9f80642bb30543f7947)* ## Summary by CodeRabbit * **New Features** * Claude Sonnet 5.5 is now available in the model picker and integrations, with adaptive thinking controls from low to max. * Claude Sonnet 5 is listed with adaptive thinking options, and model documentation includes updated reasoning settings. * **Documentation** * Updated model guides and release information with Claude Sonnet 5.5 availability and model details. * Clarified that older sandbox images or resumed sessions may not support newer models. --------- Co-authored-by: Cole Murray <2492022+ColeMurray@users.noreply.github.com> Co-authored-by: waclaude --- CHANGELOG.md | 4 ++ README.md | 11 ++--- docs/AVAILABLE_MODELS.md | 9 ++-- docs/CLAUDE_AGENT.md | 8 ++-- .../content/docs/models/choosing-a-model.mdx | 3 +- packages/modal-infra/uv.lock | 15 ++++--- packages/sandbox-images/locks/runtime.txt | 13 +++--- packages/sandbox-runtime/pyproject.toml | 2 +- .../tests/fixtures/reasoning-models.json | 30 +++++++++++++ .../tests/test_claude_harness.py | 4 +- .../tests/test_opencode_reasoning_contract.py | 4 +- packages/sandbox-runtime/uv.lock | 15 ++++--- packages/shared/src/models.test.ts | 42 +++++++++++++++++++ packages/shared/src/models.ts | 11 ++++- 14 files changed, 125 insertions(+), 46 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b0ae09f966..2cb76e94c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,10 @@ a team lead, or a workspace administrator. ### Added +**Claude Sonnet 5.5.** Adds `anthropic/claude-sonnet-5-5` to the model picker and integrations, with +adaptive thinking controls from low through max. Claude Agent SDK 0.2.161 bundles Claude Code +2.1.284, which supports the new model. + OpenCode sessions using a connected ChatGPT subscription now report estimated model costs through the existing session cost display and spending limit. These are API-price equivalents, not additional subscription charges or an OpenAI invoice; estimates remain zero if catalog pricing is diff --git a/README.md b/README.md index 8f869e25fd..55fb6e5827 100644 --- a/README.md +++ b/README.md @@ -209,19 +209,14 @@ await configureGitIdentity({ Choose the AI model that fits your task, with per-session reasoning effort controls: -| Provider | Models | -| ---------------- | ----------------------------------------------------------------------- | -| Anthropic | Claude Haiku 4.5, Sonnet 4.5/4.6/5, Opus 4.5/4.6/4.7/4.8/5, Fable 5/5.1 | -| OpenAI | GPT 5.4, GPT 5.5, 5.3 Codex, 5.3 Codex Spark | -| xAI / SuperGrok | Grok models (opt-in) | -| OpenCode Zen | Kimi K2.5/K2.6/K3, MiniMax M2.5, Qwen3.7 Max, GLM 5/5.1/5.2 (opt-in) | -| Z.AI Coding Plan | GLM 5.2/5.3 (opt-in) | +Anthropic and OpenAI models are enabled by default. xAI / SuperGrok, OpenCode Zen and Go, Z.AI +Coding Plan, and DeepSeek models are opt-in. See [Available Models](docs/AVAILABLE_MODELS.md) for +current model IDs, descriptions, and reasoning efforts. OpenAI models work with your existing ChatGPT subscription via OAuth — no separate API key needed. Anthropic models can run on the **Claude Agent** harness with a connected Claude subscription; see [Using the Claude Agent Harness](docs/CLAUDE_AGENT.md). Grok models work with an eligible SuperGrok subscription through control-plane-managed OAuth. See -**[docs/AVAILABLE_MODELS.md](docs/AVAILABLE_MODELS.md)** for the full model list and **[docs/OPENAI_MODELS.md](docs/OPENAI_MODELS.md)** or **[docs/GROK_MODELS.md](docs/GROK_MODELS.md)** for subscription setup instructions. diff --git a/docs/AVAILABLE_MODELS.md b/docs/AVAILABLE_MODELS.md index 33eba020f0..e7b3b135bd 100644 --- a/docs/AVAILABLE_MODELS.md +++ b/docs/AVAILABLE_MODELS.md @@ -35,7 +35,8 @@ Accounts) applies only on the Claude Agent harness; OpenCode sessions use `ANTHR | `anthropic/claude-haiku-4-5` | Claude Haiku 4.5 | Fast and efficient | high, max | max | | `anthropic/claude-sonnet-4-5` | Claude Sonnet 4.5 | Balanced performance | high, max | max | | `anthropic/claude-sonnet-4-6` | Claude Sonnet 4.6 | Balanced, fast coding | low, medium, high, max | high | -| `anthropic/claude-sonnet-5` | Claude Sonnet 5 | Latest Sonnet, adaptive thinking | low, medium, high, xhigh, max | high | +| `anthropic/claude-sonnet-5` | Claude Sonnet 5 | Balanced performance, adaptive thinking | low, medium, high, xhigh, max | high | +| `anthropic/claude-sonnet-5-5` | Claude Sonnet 5.5 | Latest Sonnet, fast and intelligent | low, medium, high, xhigh, max | high | | `anthropic/claude-opus-4-5` | Claude Opus 4.5 | Most capable | high, max | max | | `anthropic/claude-opus-4-6` | Claude Opus 4.6 | Most capable, adaptive thinking | low, medium, high, max | high | | `anthropic/claude-opus-4-7` | Claude Opus 4.7 | Most capable, adaptive thinking | low, medium, high, xhigh, max | high | @@ -54,9 +55,9 @@ OpenAI models support connected ChatGPT provider accounts or `OPENAI_API_KEY` mo | ---------------------- | ------------- | ---------------------------------------------- | ----------------------------------- | -------------- | | `openai/gpt-5.4` | GPT 5.4 | Flagship model | none, low, medium, high, xhigh | Not set | | `openai/gpt-5.5` | GPT 5.5 | Latest flagship model | none, low, medium, high, xhigh | Not set | -| `openai/gpt-5.6-sol` | GPT 5.6 Sol | Frontier model for complex professional work | none, low, medium, high, xhigh | Not set | -| `openai/gpt-5.6-terra` | GPT 5.6 Terra | Balanced, cost-efficient everyday work | none, low, medium, high, xhigh | Not set | -| `openai/gpt-5.6-luna` | GPT 5.6 Luna | Fast, cost-efficient high-volume workloads | none, low, medium, high, xhigh | Not set | +| `openai/gpt-5.6-sol` | GPT 5.6 Sol | Frontier model for complex professional work | none, low, medium, high, xhigh | medium | +| `openai/gpt-5.6-terra` | GPT 5.6 Terra | Balanced, cost-efficient everyday work | none, low, medium, high, xhigh | medium | +| `openai/gpt-5.6-luna` | GPT 5.6 Luna | Fast, cost-efficient high-volume workloads | none, low, medium, high, xhigh, max | medium | | `openai/gpt-6-astra` | GPT-6 Astra | Most capable model for complex, demanding work | low, medium, high, xhigh, max | medium | | `openai/gpt-6-sol` | GPT-6 Sol | Complex coding and agentic workflows | none, low, medium, high, xhigh, max | medium | | `openai/gpt-6-luna` | GPT-6 Luna | Efficient model for focused, high-volume tasks | none, low, medium, high, xhigh, max | medium | diff --git a/docs/CLAUDE_AGENT.md b/docs/CLAUDE_AGENT.md index 48202c57b5..f4e4bb364d 100644 --- a/docs/CLAUDE_AGENT.md +++ b/docs/CLAUDE_AGENT.md @@ -196,8 +196,6 @@ fix instead. guard, the harness ignores the result of any turn it did not submit. - **Follow-ups queue.** Both harnesses hold follow-up prompts until the running turn completes. - **Image.** The sandbox image pins `claude-agent-sdk`, whose wheel bundles the `claude` binary. The - runtime manifest names the generation carrying the current pin under `harnessMinimumGeneration`, - so a Claude session never boots a prebuilt image from before that generation; this floor does not - touch OpenCode sessions' images or snapshots, since the global compatibility floor did not move. - Raise this floor whenever the SDK pin moves for a model the catalog advertises, otherwise a - session can be handed an older image whose bundled `claude` does not know that model. + runtime manifest's `harnessMinimumGeneration` controls which prepared images new Claude sessions + can use. Older images and resumed snapshots may lack newer models until rebuilt; a model request + can fail on a sandbox whose bundled CLI does not support it. diff --git a/packages/docs/content/docs/models/choosing-a-model.mdx b/packages/docs/content/docs/models/choosing-a-model.mdx index c7ab6f11c9..ce264a973e 100644 --- a/packages/docs/content/docs/models/choosing-a-model.mdx +++ b/packages/docs/content/docs/models/choosing-a-model.mdx @@ -4,7 +4,7 @@ description: Enable models for your workspace, pick a model and effort in the co audience: team-owner owner: platform status: published -lastReviewed: "2026-09-23" +lastReviewed: "2026-09-28" relatedCode: - packages/shared/src/models.ts - packages/shared/src/harnesses.ts @@ -62,6 +62,7 @@ Runs on both harnesses. Credential: `ANTHROPIC_API_KEY` as a global secret, or o | `anthropic/claude-sonnet-4-5` | Claude Sonnet 4.5 | high, max | max | | `anthropic/claude-sonnet-4-6` | Claude Sonnet 4.6 | low, medium, high, max | high | | `anthropic/claude-sonnet-5` | Claude Sonnet 5 | low, medium, high, xhigh, max | high | +| `anthropic/claude-sonnet-5-5` | Claude Sonnet 5.5 | low, medium, high, xhigh, max | high | | `anthropic/claude-opus-4-5` | Claude Opus 4.5 | high, max | max | | `anthropic/claude-opus-4-6` | Claude Opus 4.6 | low, medium, high, max | high | | `anthropic/claude-opus-4-7` | Claude Opus 4.7 | low, medium, high, xhigh, max | high | diff --git a/packages/modal-infra/uv.lock b/packages/modal-infra/uv.lock index f07eebf169..830fc07f44 100644 --- a/packages/modal-infra/uv.lock +++ b/packages/modal-infra/uv.lock @@ -266,7 +266,7 @@ wheels = [ [[package]] name = "claude-agent-sdk" -version = "0.2.158" +version = "0.2.161" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, @@ -274,13 +274,12 @@ dependencies = [ { name = "mcp" }, { name = "sniffio" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/0e/5c/1d348e677fb199978d2b559fb5822fc58d77927f0988d1eb2acc18d5a0bb/claude_agent_sdk-0.2.158.tar.gz", hash = "sha256:672ddb19df334e9cb80bb264a6ddf00870c8d4e170b3fd0a3cf0ce354f634e7f", size = 354570, upload-time = "2026-09-23T01:39:14.77Z" } +sdist = { url = "https://files.pythonhosted.org/packages/8b/e1/8c94ff7f200eabcee9a9e57825dfac596bb6fbdeeafa594f49bc7e7f89b5/claude_agent_sdk-0.2.161.tar.gz", hash = "sha256:284a1372ec3af133578009ab2b821a238272dbf17d687a58be9df918d7b3ae89", size = 365168, upload-time = "2026-09-28T18:53:39.197Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/89/ad/37559b5a12170699eb6d4c5e1c66f491c4a3d7366f8f300a8dba0fc40aec/claude_agent_sdk-0.2.158-py3-none-macosx_11_0_arm64.whl", hash = "sha256:d124bd374dfbbcbdb4326e922f30d436251ae469df9cae545ef8ff6839d76e47", size = 90406616, upload-time = "2026-09-23T01:39:18.829Z" }, - { url = "https://files.pythonhosted.org/packages/c1/52/21968fc645ee206bb260f18b26ae8ae035fc5af820a7820b01e308496510/claude_agent_sdk-0.2.158-py3-none-macosx_11_0_x86_64.whl", hash = "sha256:beb963aa66eac715ff005e44ff02065fd5d855a10a47bc802097a31a522e6be3", size = 94751713, upload-time = "2026-09-23T01:39:22.591Z" }, - { url = "https://files.pythonhosted.org/packages/a1/4b/b42ab1a7ba32cea66184dedf8da158d49d00d6af753bd48998dab6e7a1d6/claude_agent_sdk-0.2.158-py3-none-manylinux_2_17_aarch64.whl", hash = "sha256:12f432273c3f0da103ea8dc921c3ce96e802c044e1f3f2535cf98b7d59777aa7", size = 99897552, upload-time = "2026-09-23T01:39:26.933Z" }, - { url = "https://files.pythonhosted.org/packages/e3/eb/1b34083c59a09a1a380c211eacd1cb185de4adc4bdb1170997e37f53f6ca/claude_agent_sdk-0.2.158-py3-none-manylinux_2_17_x86_64.whl", hash = "sha256:1f46ddac2cb51f16b2ca064a0dbeeab7db712f34e72454cc7776713e0f0f663c", size = 100257803, upload-time = "2026-09-23T01:39:31.501Z" }, - { url = "https://files.pythonhosted.org/packages/15/0d/11ab53934b04176180099a79bb186036e304e1fe1310bdecf21df599f1d3/claude_agent_sdk-0.2.158-py3-none-win_amd64.whl", hash = "sha256:b1e7f6b9d048c2c44e5228d6a0eeb19ae3aafb643c1115c456ea9937e0439aef", size = 102853711, upload-time = "2026-09-23T01:39:36.133Z" }, + { url = "https://files.pythonhosted.org/packages/e8/be/528e3ec2bb30deff4fcc9484dd47d1022786ad3659d31110bd0ea66ea548/claude_agent_sdk-0.2.161-py3-none-macosx_11_0_arm64.whl", hash = "sha256:127c18a790dc9f77cc2d648f932e11b06cd3325c4b3de7a73d3bcf3d3b74b8bc", size = 93872213, upload-time = "2026-09-28T18:53:42.956Z" }, + { url = "https://files.pythonhosted.org/packages/17/1c/a0dcb628ac2f25bc92d858857a399202723098057828a909ca4f9893b721/claude_agent_sdk-0.2.161-py3-none-macosx_11_0_x86_64.whl", hash = "sha256:599e1c8ccb4d48ed57999b48e80937d6a6aefbc52b0c0cda1291aea21df29e3e", size = 98316495, upload-time = "2026-09-28T18:53:47.239Z" }, + { url = "https://files.pythonhosted.org/packages/73/04/3c0caee46c27eee75642b52ee11857c0d3b9d6917ecfd916da2ffc2c2d3f/claude_agent_sdk-0.2.161-py3-none-manylinux_2_17_aarch64.whl", hash = "sha256:18081d3277c07a71891bd15d1496387d9efc5f04a6b482ae3a03e7a0b8067069", size = 103331951, upload-time = "2026-09-28T18:53:51.367Z" }, + { url = "https://files.pythonhosted.org/packages/09/70/2c7c4160a19c6edd42e317bdc9cf810e91b6a535c04377fb09d4ee090769/claude_agent_sdk-0.2.161-py3-none-manylinux_2_17_x86_64.whl", hash = "sha256:b854e46d2889a51e1a77978711a05c757e63064bd5d8f89bcf74bb507babfc2c", size = 103684983, upload-time = "2026-09-28T18:53:55.496Z" }, ] [[package]] @@ -951,7 +950,7 @@ dependencies = [ [package.metadata] requires-dist = [ - { name = "claude-agent-sdk", specifier = "==0.2.158" }, + { name = "claude-agent-sdk", specifier = "==0.2.161" }, { name = "cryptography", specifier = ">=44.0.0" }, { name = "httpx", specifier = ">=0.27.0" }, { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.14.0" }, diff --git a/packages/sandbox-images/locks/runtime.txt b/packages/sandbox-images/locks/runtime.txt index 8f6978407f..d20a2d4dbf 100644 --- a/packages/sandbox-images/locks/runtime.txt +++ b/packages/sandbox-images/locks/runtime.txt @@ -86,13 +86,12 @@ cffi==2.1.1 ; platform_python_implementation != 'PyPy' \ --hash=sha256:f81b3b8f3d4e343550fa4baa0e479bba9f2d29ce9c2e9b51d1ce1718d7442fcf \ --hash=sha256:fb92203a88b3d3053034db775110081c49d28be6551923805e039924093761e4 \ --hash=sha256:fcd22650c908d7b7da162bbfaab594a1227a15d1643a98c68b122ac642fa2264 -claude-agent-sdk==0.2.158 \ - --hash=sha256:12f432273c3f0da103ea8dc921c3ce96e802c044e1f3f2535cf98b7d59777aa7 \ - --hash=sha256:1f46ddac2cb51f16b2ca064a0dbeeab7db712f34e72454cc7776713e0f0f663c \ - --hash=sha256:672ddb19df334e9cb80bb264a6ddf00870c8d4e170b3fd0a3cf0ce354f634e7f \ - --hash=sha256:b1e7f6b9d048c2c44e5228d6a0eeb19ae3aafb643c1115c456ea9937e0439aef \ - --hash=sha256:beb963aa66eac715ff005e44ff02065fd5d855a10a47bc802097a31a522e6be3 \ - --hash=sha256:d124bd374dfbbcbdb4326e922f30d436251ae469df9cae545ef8ff6839d76e47 +claude-agent-sdk==0.2.161 \ + --hash=sha256:127c18a790dc9f77cc2d648f932e11b06cd3325c4b3de7a73d3bcf3d3b74b8bc \ + --hash=sha256:18081d3277c07a71891bd15d1496387d9efc5f04a6b482ae3a03e7a0b8067069 \ + --hash=sha256:284a1372ec3af133578009ab2b821a238272dbf17d687a58be9df918d7b3ae89 \ + --hash=sha256:599e1c8ccb4d48ed57999b48e80937d6a6aefbc52b0c0cda1291aea21df29e3e \ + --hash=sha256:b854e46d2889a51e1a77978711a05c757e63064bd5d8f89bcf74bb507babfc2c click==8.5.0 ; sys_platform != 'emscripten' \ --hash=sha256:255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360 \ --hash=sha256:ba0d2089de75ea0310e2dde03160e6ca10009947fb95a182f9b54021bb272e34 diff --git a/packages/sandbox-runtime/pyproject.toml b/packages/sandbox-runtime/pyproject.toml index 5325d57fb3..6bf50edd44 100644 --- a/packages/sandbox-runtime/pyproject.toml +++ b/packages/sandbox-runtime/pyproject.toml @@ -11,7 +11,7 @@ dependencies = [ "PyJWT[crypto]>=2.9.0", # Exact pin: the wheel bundles the `claude` binary and its message shapes # are what harness/claude.py translates. - "claude-agent-sdk==0.2.158", + "claude-agent-sdk==0.2.161", ] [project.optional-dependencies] diff --git a/packages/sandbox-runtime/tests/fixtures/reasoning-models.json b/packages/sandbox-runtime/tests/fixtures/reasoning-models.json index baa061f026..706458fcbf 100644 --- a/packages/sandbox-runtime/tests/fixtures/reasoning-models.json +++ b/packages/sandbox-runtime/tests/fixtures/reasoning-models.json @@ -132,6 +132,36 @@ "cache_write": 2.5 } }, + "claude-sonnet-5-5": { + "id": "claude-sonnet-5-5", + "name": "Claude Sonnet 5.5", + "family": "claude-sonnet", + "attachment": true, + "reasoning": true, + "reasoning_options": [ + { + "type": "effort", + "values": ["low", "medium", "high", "xhigh", "max"] + } + ], + "tool_call": true, + "temperature": false, + "release_date": "2026-09-28", + "modalities": { + "input": ["text", "image", "pdf"], + "output": ["text"] + }, + "limit": { + "context": 1000000, + "output": 128000 + }, + "cost": { + "input": 2, + "output": 10, + "cache_read": 0.2, + "cache_write": 2.5 + } + }, "claude-opus-4-5": { "id": "claude-opus-4-5", "name": "Claude Opus 4.5 (latest)", diff --git a/packages/sandbox-runtime/tests/test_claude_harness.py b/packages/sandbox-runtime/tests/test_claude_harness.py index ed0918791a..826f7708e4 100644 --- a/packages/sandbox-runtime/tests/test_claude_harness.py +++ b/packages/sandbox-runtime/tests/test_claude_harness.py @@ -325,14 +325,14 @@ async def test_options_follow_the_design_mapping(self, tmp_path: Path) -> None: HarnessPrompt( message_id="m1", text="hi", - model="anthropic/claude-opus-4-6", + model="anthropic/claude-sonnet-5-5", reasoning_effort="high", ), ) options = h.client.options assert options["cwd"] == str(tmp_path / "repo") assert options["cli_path"] == str(h.harness.wrapper_path) - assert options["model"] == "claude-opus-4-6" + assert options["model"] == "claude-sonnet-5-5" assert options["effort"] == "high" assert options["permission_mode"] == "dontAsk" assert options["disallowed_tools"] == ["AskUserQuestion"] diff --git a/packages/sandbox-runtime/tests/test_opencode_reasoning_contract.py b/packages/sandbox-runtime/tests/test_opencode_reasoning_contract.py index 7e12bdbb27..8fda795110 100644 --- a/packages/sandbox-runtime/tests/test_opencode_reasoning_contract.py +++ b/packages/sandbox-runtime/tests/test_opencode_reasoning_contract.py @@ -9,7 +9,9 @@ Source SHA-256: c9d3dc07540cf91a7b7362a3b42943132965f4ee6e8374113ee4f0f7b56b3d90 Claude Opus 5.5 added from the 2026-09-23 retrieval. Source SHA-256: e20acec396a73dc3db45d0eca7f0ede5bff28f09f002ba96ce7b1b566de7b6d0 -Subset SHA-256: e2b3a03acdfc3551f2e1752e26578768fa3265374b0fc79432392e4ce01d061d +Claude Sonnet 5.5 added from the 2026-09-28 retrieval. +Source SHA-256: 06e0071dd4ae9c9da2db1fabf28eb4994914fefdc5dd10270a5b340c88a49aec +Subset SHA-256: e9c9cc6f90fa9afbc75a2f18bf564398594d3f51693667cf1efd229617aaab0b Reconcile this frozen fixture with shared model/effort definitions when changing models or the binary. Mocks verify serialization, not live provider acceptance. """ diff --git a/packages/sandbox-runtime/uv.lock b/packages/sandbox-runtime/uv.lock index 064d54ce39..9ca7863f41 100644 --- a/packages/sandbox-runtime/uv.lock +++ b/packages/sandbox-runtime/uv.lock @@ -175,7 +175,7 @@ wheels = [ [[package]] name = "claude-agent-sdk" -version = "0.2.158" +version = "0.2.161" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, @@ -183,13 +183,12 @@ dependencies = [ { name = "mcp" }, { name = "sniffio" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/0e/5c/1d348e677fb199978d2b559fb5822fc58d77927f0988d1eb2acc18d5a0bb/claude_agent_sdk-0.2.158.tar.gz", hash = "sha256:672ddb19df334e9cb80bb264a6ddf00870c8d4e170b3fd0a3cf0ce354f634e7f", size = 354570, upload-time = "2026-09-23T01:39:14.77Z" } +sdist = { url = "https://files.pythonhosted.org/packages/8b/e1/8c94ff7f200eabcee9a9e57825dfac596bb6fbdeeafa594f49bc7e7f89b5/claude_agent_sdk-0.2.161.tar.gz", hash = "sha256:284a1372ec3af133578009ab2b821a238272dbf17d687a58be9df918d7b3ae89", size = 365168, upload-time = "2026-09-28T18:53:39.197Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/89/ad/37559b5a12170699eb6d4c5e1c66f491c4a3d7366f8f300a8dba0fc40aec/claude_agent_sdk-0.2.158-py3-none-macosx_11_0_arm64.whl", hash = "sha256:d124bd374dfbbcbdb4326e922f30d436251ae469df9cae545ef8ff6839d76e47", size = 90406616, upload-time = "2026-09-23T01:39:18.829Z" }, - { url = "https://files.pythonhosted.org/packages/c1/52/21968fc645ee206bb260f18b26ae8ae035fc5af820a7820b01e308496510/claude_agent_sdk-0.2.158-py3-none-macosx_11_0_x86_64.whl", hash = "sha256:beb963aa66eac715ff005e44ff02065fd5d855a10a47bc802097a31a522e6be3", size = 94751713, upload-time = "2026-09-23T01:39:22.591Z" }, - { url = "https://files.pythonhosted.org/packages/a1/4b/b42ab1a7ba32cea66184dedf8da158d49d00d6af753bd48998dab6e7a1d6/claude_agent_sdk-0.2.158-py3-none-manylinux_2_17_aarch64.whl", hash = "sha256:12f432273c3f0da103ea8dc921c3ce96e802c044e1f3f2535cf98b7d59777aa7", size = 99897552, upload-time = "2026-09-23T01:39:26.933Z" }, - { url = "https://files.pythonhosted.org/packages/e3/eb/1b34083c59a09a1a380c211eacd1cb185de4adc4bdb1170997e37f53f6ca/claude_agent_sdk-0.2.158-py3-none-manylinux_2_17_x86_64.whl", hash = "sha256:1f46ddac2cb51f16b2ca064a0dbeeab7db712f34e72454cc7776713e0f0f663c", size = 100257803, upload-time = "2026-09-23T01:39:31.501Z" }, - { url = "https://files.pythonhosted.org/packages/15/0d/11ab53934b04176180099a79bb186036e304e1fe1310bdecf21df599f1d3/claude_agent_sdk-0.2.158-py3-none-win_amd64.whl", hash = "sha256:b1e7f6b9d048c2c44e5228d6a0eeb19ae3aafb643c1115c456ea9937e0439aef", size = 102853711, upload-time = "2026-09-23T01:39:36.133Z" }, + { url = "https://files.pythonhosted.org/packages/e8/be/528e3ec2bb30deff4fcc9484dd47d1022786ad3659d31110bd0ea66ea548/claude_agent_sdk-0.2.161-py3-none-macosx_11_0_arm64.whl", hash = "sha256:127c18a790dc9f77cc2d648f932e11b06cd3325c4b3de7a73d3bcf3d3b74b8bc", size = 93872213, upload-time = "2026-09-28T18:53:42.956Z" }, + { url = "https://files.pythonhosted.org/packages/17/1c/a0dcb628ac2f25bc92d858857a399202723098057828a909ca4f9893b721/claude_agent_sdk-0.2.161-py3-none-macosx_11_0_x86_64.whl", hash = "sha256:599e1c8ccb4d48ed57999b48e80937d6a6aefbc52b0c0cda1291aea21df29e3e", size = 98316495, upload-time = "2026-09-28T18:53:47.239Z" }, + { url = "https://files.pythonhosted.org/packages/73/04/3c0caee46c27eee75642b52ee11857c0d3b9d6917ecfd916da2ffc2c2d3f/claude_agent_sdk-0.2.161-py3-none-manylinux_2_17_aarch64.whl", hash = "sha256:18081d3277c07a71891bd15d1496387d9efc5f04a6b482ae3a03e7a0b8067069", size = 103331951, upload-time = "2026-09-28T18:53:51.367Z" }, + { url = "https://files.pythonhosted.org/packages/09/70/2c7c4160a19c6edd42e317bdc9cf810e91b6a535c04377fb09d4ee090769/claude_agent_sdk-0.2.161-py3-none-manylinux_2_17_x86_64.whl", hash = "sha256:b854e46d2889a51e1a77978711a05c757e63064bd5d8f89bcf74bb507babfc2c", size = 103684983, upload-time = "2026-09-28T18:53:55.496Z" }, ] [[package]] @@ -558,7 +557,7 @@ dev = [ [package.metadata] requires-dist = [ - { name = "claude-agent-sdk", specifier = "==0.2.158" }, + { name = "claude-agent-sdk", specifier = "==0.2.161" }, { name = "cryptography", specifier = ">=44.0.0" }, { name = "httpx", specifier = ">=0.27.0" }, { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.14.0" }, diff --git a/packages/shared/src/models.test.ts b/packages/shared/src/models.test.ts index 3697f7ecef..390bd3ef45 100644 --- a/packages/shared/src/models.test.ts +++ b/packages/shared/src/models.test.ts @@ -1,3 +1,4 @@ +import { readFileSync } from "node:fs"; import { describe, expect, it } from "vitest"; import { DEFAULT_ENABLED_MODELS, @@ -25,6 +26,7 @@ const ANTHROPIC_MODELS = [ "anthropic/claude-sonnet-4-5", "anthropic/claude-sonnet-4-6", "anthropic/claude-sonnet-5", + "anthropic/claude-sonnet-5-5", "anthropic/claude-opus-4-5", "anthropic/claude-opus-4-6", "anthropic/claude-opus-4-7", @@ -92,6 +94,37 @@ const GO_MODELS = [ const DEEPSEEK_MODELS = ["deepseek/deepseek-v4-flash", "deepseek/deepseek-v4-pro"] as const; const ZAI_CODING_PLAN_MODELS = ["zai-coding-plan/glm-5.2", "zai-coding-plan/glm-5.3"] as const; +it.each([ + "../../../docs/AVAILABLE_MODELS.md", + "../../../packages/docs/content/docs/models/choosing-a-model.mdx", +])("keeps the model inventory in %s aligned with the catalog", (file) => { + const rows = readFileSync(new URL(file, import.meta.url), "utf8") + .split("\n") + .filter((line) => /^\| `[^`]+`/.test(line)) + .map((line) => + line + .split("|") + .slice(1, -1) + .map((cell) => cell.trim()) + ); + const models = MODEL_CATALOG.flatMap((group) => group.models); + + expect(rows.map(([id]) => id).sort()).toEqual(models.map(({ id }) => `\`${id}\``).sort()); + for (const model of models) { + const row = rows.find(([id]) => id === `\`${model.id}\``); + expect(row?.[1]).toBe(model.name); + if (file.endsWith("AVAILABLE_MODELS.md")) { + expect(row?.[2]).toBe(model.description); + } + if ("reasoning" in model) { + expect(row?.at(-2)).toBe(model.reasoning.efforts.join(", ")); + expect(row?.at(-1)).toBe(model.reasoning.default ?? "Not set"); + } else { + expect(row?.at(-1)).toBe("N/A"); + } + } +}); + describe("model utilities", () => { it("derives every public model view from the authoritative catalog", () => { const catalogModels = MODEL_CATALOG.flatMap((group) => group.models); @@ -151,6 +184,7 @@ describe("model utilities", () => { expect(normalizeModelId("claude-sonnet-4-6")).toBe("anthropic/claude-sonnet-4-6"); expect(normalizeModelId("claude-opus-4-8")).toBe("anthropic/claude-opus-4-8"); expect(normalizeModelId("claude-opus-5")).toBe("anthropic/claude-opus-5"); + expect(normalizeModelId("claude-sonnet-5-5")).toBe("anthropic/claude-sonnet-5-5"); expect(normalizeModelId("claude-fable-5")).toBe("anthropic/claude-fable-5"); expect(normalizeModelId("claude-fable-5-1")).toBe("anthropic/claude-fable-5-1"); expect(normalizeModelId("gpt-6-sol")).toBe("openai/gpt-6-sol"); @@ -158,6 +192,7 @@ describe("model utilities", () => { expect(isValidModel("claude-sonnet-4-6")).toBe(true); expect(isValidModel("claude-opus-4-8")).toBe(true); expect(isValidModel("claude-opus-5")).toBe(true); + expect(isValidModel("claude-sonnet-5-5")).toBe(true); expect(isValidModel("claude-fable-5")).toBe(true); expect(isValidModel("claude-fable-5-1")).toBe(true); expect(isValidModel("gpt-6-sol")).toBe(true); @@ -355,6 +390,7 @@ describe("model utilities", () => { expect(getDefaultReasoningEffort("anthropic/claude-sonnet-4-6")).toBe("high"); expect(getDefaultReasoningEffort("anthropic/claude-opus-4-8")).toBe("high"); expect(getDefaultReasoningEffort("anthropic/claude-sonnet-5")).toBe("high"); + expect(getDefaultReasoningEffort("anthropic/claude-sonnet-5-5")).toBe("high"); expect(getDefaultReasoningEffort("anthropic/claude-opus-5")).toBe("high"); expect(getDefaultReasoningEffort("anthropic/claude-opus-5-5")).toBe("high"); expect(getDefaultReasoningEffort("anthropic/claude-fable-5")).toBe("high"); @@ -381,6 +417,10 @@ describe("model utilities", () => { efforts: ["low", "medium", "high", "xhigh", "max"], default: "high", }); + expect(getReasoningConfig("anthropic/claude-sonnet-5-5")).toEqual({ + efforts: ["low", "medium", "high", "xhigh", "max"], + default: "high", + }); expect(getReasoningConfig("anthropic/claude-opus-4-8")).toEqual({ efforts: ["low", "medium", "high", "xhigh", "max"], default: "high", @@ -448,6 +488,8 @@ describe("model utilities", () => { expect(isValidReasoningEffort("anthropic/claude-opus-4-8", "xhigh")).toBe(true); expect(isValidReasoningEffort("anthropic/claude-opus-4-8", "none")).toBe(false); expect(isValidReasoningEffort("anthropic/claude-sonnet-5", "xhigh")).toBe(true); + expect(isValidReasoningEffort("anthropic/claude-sonnet-5-5", "max")).toBe(true); + expect(isValidReasoningEffort("anthropic/claude-sonnet-5-5", "none")).toBe(false); expect(isValidReasoningEffort("anthropic/claude-opus-5", "xhigh")).toBe(true); expect(isValidReasoningEffort("anthropic/claude-opus-5", "none")).toBe(false); expect(isValidReasoningEffort("anthropic/claude-opus-5-5", "xhigh")).toBe(true); diff --git a/packages/shared/src/models.ts b/packages/shared/src/models.ts index 385f5980ed..d06e76b856 100644 --- a/packages/shared/src/models.ts +++ b/packages/shared/src/models.ts @@ -70,7 +70,16 @@ export const MODEL_CATALOG = [ { id: "anthropic/claude-sonnet-5", name: "Claude Sonnet 5", - description: "Latest Sonnet, adaptive thinking", + description: "Balanced performance, adaptive thinking", + reasoning: { + efforts: ["low", "medium", "high", "xhigh", "max"], + default: "high", + }, + }, + { + id: "anthropic/claude-sonnet-5-5", + name: "Claude Sonnet 5.5", + description: "Latest Sonnet, fast and intelligent", reasoning: { efforts: ["low", "medium", "high", "xhigh", "max"], default: "high", From bf7e023779d5ceab8c18032f8249c1b3d4794473 Mon Sep 17 00:00:00 2001 From: Vaibhav Hajare <166456890+lab1207@users.noreply.github.com> Date: Tue, 29 Sep 2026 09:29:46 +0530 Subject: [PATCH 06/13] fix(ci): disable persisted credentials in checkout steps (#2130) Closes #1546 Sets persist-credentials:false on all 9 actions/checkout steps in .github/workflows/ci.yml (7 added, 2 already present). Verification: - python yaml.safe_load passes - 9 checkout steps, 9 persist-credentials:false - No push/release jobs affected, permissions contents:read ## Summary by CodeRabbit * **Security** * Checkout credentials are no longer persisted across the affected lint, typecheck, build, and test jobs. --- .github/workflows/ci.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 77a7c68aaa..ee8d68fd86 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -93,6 +93,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v6 + with: + persist-credentials: false - name: Setup Node.js uses: actions/setup-node@v6 @@ -148,6 +150,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v6 + with: + persist-credentials: false - name: Setup Node.js uses: actions/setup-node@v6 @@ -171,6 +175,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v6 + with: + persist-credentials: false - name: Setup Node.js uses: actions/setup-node@v6 @@ -218,6 +224,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v6 + with: + persist-credentials: false - name: Setup Node.js uses: actions/setup-node@v6 @@ -251,6 +259,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v6 + with: + persist-credentials: false - name: Setup Node.js uses: actions/setup-node@v6 @@ -276,6 +286,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v6 + with: + persist-credentials: false - name: Setup Node.js uses: actions/setup-node@v6 @@ -321,6 +333,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v6 + with: + persist-credentials: false - name: Setup Node.js uses: actions/setup-node@v6 From b027b4d3708db0eac0e5a57415059eb87d9bd9ac Mon Sep 17 00:00:00 2001 From: Cole Murray Date: Mon, 28 Sep 2026 21:01:56 -0700 Subject: [PATCH 07/13] fix: disable Bun fetch timeout for Codex OAuth streams (#2129) ## Summary - Disable the Bun fetch idle timeout for Codex OAuth proxied requests so OpenCode owns stalled-stream handling and retry classification. - Assert that the upstream fetch receives `timeout: false`. - Pin Modal to the tested 1.4.3 SDK version: CI otherwise installs 1.6.0, whose removal of endpoint introspection APIs breaks unrelated modal-infra tests. ## Validation - `node --test packages/sandbox-runtime/tests/codex-auth-plugin.test.mjs` (5 passed) - `bun test packages/sandbox-runtime/tests/codex-auth-plugin.test.mjs` (5 passed) - `uv run pytest tests/ -q` in `packages/modal-infra` (350 passed) - `uv lock --check` and `uv run ruff check src/ tests/` / `uv run ruff format --check src/ tests/` in `packages/modal-infra` passed. Long-stall OpenCode retry behavior was not tested end to end. ## Summary by CodeRabbit * **Bug Fixes** * Proxied Codex authentication requests no longer use the default fetch timeout. Requests can continue beyond that timeout window rather than being interrupted by it, helping avoid unexpected timeouts when the authentication service takes longer to respond. This change applies specifically to requests forwarded through the Codex authentication proxy; other authentication behavior is unchanged. --------- Co-authored-by: Cole Murray <2492022+ColeMurray@users.noreply.github.com> --- packages/modal-infra/pyproject.toml | 2 +- packages/modal-infra/uv.lock | 2 +- .../src/sandbox_runtime/plugins/codex-auth-plugin.js | 3 ++- packages/sandbox-runtime/tests/codex-auth-plugin.test.mjs | 3 +++ 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/modal-infra/pyproject.toml b/packages/modal-infra/pyproject.toml index fdb829fc6c..5e1c45bc05 100644 --- a/packages/modal-infra/pyproject.toml +++ b/packages/modal-infra/pyproject.toml @@ -5,7 +5,7 @@ description = "Modal sandbox infrastructure for Open-Inspect coding agent" requires-python = ">=3.12" dependencies = [ "open-inspect-sandbox-runtime", # sibling package, resolved via [tool.uv.sources] - "modal>=1.4.3", # Function.with_options() (per-call timeout override) requires >=1.4.3 + "modal==1.4.3", # Tested SDK; newer releases remove endpoint introspection used by tests "httpx>=0.27.0", "pydantic>=2.0", "fastapi>=0.110.0", diff --git a/packages/modal-infra/uv.lock b/packages/modal-infra/uv.lock index 830fc07f44..3857456ca3 100644 --- a/packages/modal-infra/uv.lock +++ b/packages/modal-infra/uv.lock @@ -924,7 +924,7 @@ dev = [ requires-dist = [ { name = "fastapi", specifier = ">=0.110.0" }, { name = "httpx", specifier = ">=0.27.0" }, - { name = "modal", specifier = ">=1.4.3" }, + { name = "modal", specifier = "==1.4.3" }, { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.14.0" }, { name = "open-inspect-sandbox-runtime", editable = "../sandbox-runtime" }, { name = "pydantic", specifier = ">=2.0" }, diff --git a/packages/sandbox-runtime/src/sandbox_runtime/plugins/codex-auth-plugin.js b/packages/sandbox-runtime/src/sandbox_runtime/plugins/codex-auth-plugin.js index 9c3159b067..505ed34e85 100644 --- a/packages/sandbox-runtime/src/sandbox_runtime/plugins/codex-auth-plugin.js +++ b/packages/sandbox-runtime/src/sandbox_runtime/plugins/codex-auth-plugin.js @@ -176,7 +176,8 @@ export const CodexAuthProxy = async (input) => { proxiedRequest.headers.set("authorization", `Bearer ${accessToken}`); if (accountId) proxiedRequest.headers.set("ChatGPT-Account-Id", accountId); - return fetch(proxiedRequest); + // Let OpenCode's stream timeout handle stalls instead of Bun's fetch idle timer. + return fetch(proxiedRequest, { timeout: false }); }, }; }, diff --git a/packages/sandbox-runtime/tests/codex-auth-plugin.test.mjs b/packages/sandbox-runtime/tests/codex-auth-plugin.test.mjs index 58ab388cfe..64c9b68f93 100644 --- a/packages/sandbox-runtime/tests/codex-auth-plugin.test.mjs +++ b/packages/sandbox-runtime/tests/codex-auth-plugin.test.mjs @@ -25,6 +25,7 @@ test("preserves a source Request while proxying Codex authentication", async () process.env.SANDBOX_AUTH_TOKEN = "sandbox-token"; process.env.SESSION_CONFIG = JSON.stringify({ sessionId: "session-1" }); let upstreamRequest; + let upstreamInit; globalThis.fetch = async (input, init) => { const request = input instanceof Request ? input : new Request(input, init); if (request.url.startsWith("https://control.test/")) { @@ -35,6 +36,7 @@ test("preserves a source Request while proxying Codex authentication", async () }); } upstreamRequest = request; + upstreamInit = init; return new Response(null, { status: 200 }); }; const plugin = await CodexAuthProxy({ client: { auth: { set: async () => undefined } } }); @@ -56,6 +58,7 @@ test("preserves a source Request while proxying Codex authentication", async () assert.equal(upstreamRequest.headers.get("chatgpt-account-id"), "account-1"); assert.equal(upstreamRequest.headers.get("x-request-header"), "preserved"); assert.equal(await upstreamRequest.text(), "request-body"); + assert.deepEqual(upstreamInit, { timeout: false }); }); test("preserves API-key requests if OpenAI authentication switches away from OAuth", async () => { From ec3913ffef52f7e50a13113e39eecfb004d1dbd5 Mon Sep 17 00:00:00 2001 From: "open-inspect[bot]" <255062780+open-inspect[bot]@users.noreply.github.com> Date: Mon, 28 Sep 2026 21:02:12 -0700 Subject: [PATCH 08/13] test: cover team default environment ownership (#2126) ## Summary - Adds an integration test for `TeamStore.update` default environment ownership rules. - Covers accepting a default environment owned by the team. - Covers rejecting cross-team and missing default environments while preserving the previous valid default. ## Why Team default environment selection is core configuration logic that gates which environment a team uses by default. Recent team/environment ownership changes had coverage for an empty invalid ID, but not the high-risk ownership invariant or the edge case that failed updates must not clear an existing default. ## Tests - `npm run test:integration -w @open-inspect/control-plane -- team-stores.test.ts` --- *Created with [Open-Inspect](https://open-inspect-prod.vercel.app/session/b336ed978fd99b8b47efe9ee2f266009)* Co-authored-by: waclaude --- .../test/integration/team-stores.test.ts | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/packages/control-plane/test/integration/team-stores.test.ts b/packages/control-plane/test/integration/team-stores.test.ts index c762af61b1..e88ce7f779 100644 --- a/packages/control-plane/test/integration/team-stores.test.ts +++ b/packages/control-plane/test/integration/team-stores.test.ts @@ -1,5 +1,6 @@ import { env } from "cloudflare:test"; import { beforeEach, describe, expect, it } from "vitest"; +import { EnvironmentStore, type EnvironmentRow } from "../../src/db/environments"; import { TeamSlugConflictError, TeamStore } from "../../src/db/teams"; import { TeamMembershipStore, @@ -10,6 +11,21 @@ import { cleanD1Tables } from "./cleanup"; beforeEach(cleanD1Tables); +function environmentRow(overrides: Partial): EnvironmentRow { + const now = Date.now(); + return { + id: "env_test", + name: "Test Environment", + description: null, + prebuild_enabled: 0, + channel_associations: null, + created_at: now, + updated_at: now, + owner_team_id: null, + ...overrides, + }; +} + describe("team and membership stores", () => { it("validates team rows and allows any team to be archived or restored", async () => { const store = new TeamStore(env.DB); @@ -43,6 +59,41 @@ describe("team and membership stores", () => { expect((await store.getById(team.id))?.defaultEnvironmentId).toBeNull(); }); + it("only accepts default environments owned by the team", async () => { + const teams = new TeamStore(env.DB); + const environments = new EnvironmentStore(env.DB); + const team = await teams.create({ + slug: "default-env", + name: "Default Env", + joinPolicy: "invite_only", + }); + const otherTeam = await teams.create({ + slug: "other-default-env", + name: "Other Default Env", + joinPolicy: "invite_only", + }); + await environments.create( + environmentRow({ id: "env_same_team", name: "Same Team", owner_team_id: team.id }), + [] + ); + await environments.create( + environmentRow({ id: "env_other_team", name: "Other Team", owner_team_id: otherTeam.id }), + [] + ); + + await expect( + teams.update(team.id, { defaultEnvironmentId: "env_same_team" }) + ).resolves.toMatchObject({ defaultEnvironmentId: "env_same_team" }); + + await expect(teams.update(team.id, { defaultEnvironmentId: "env_other_team" })).rejects.toThrow( + "Default environment must belong to the team" + ); + await expect(teams.update(team.id, { defaultEnvironmentId: "env_missing" })).rejects.toThrow( + "Default environment must belong to the team" + ); + expect((await teams.getById(team.id))?.defaultEnvironmentId).toBe("env_same_team"); + }); + it("reports duplicate slugs as a typed store conflict", async () => { const store = new TeamStore(env.DB); await store.create({ slug: "duplicate", name: "First", joinPolicy: "invite_only" }); From 45007cb0572409f50a1f9ba5409803915d443927 Mon Sep 17 00:00:00 2001 From: "open-inspect[bot]" <255062780+open-inspect[bot]@users.noreply.github.com> Date: Mon, 28 Sep 2026 21:42:48 -0700 Subject: [PATCH 09/13] fix(control-plane): filter hidden children from session lists (#2131) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - Filter `GET /sessions/:id/children` through the existing per-session admission evaluator before serializing each child. For user/service requests, children the viewer cannot see are omitted: private visibility applies in every mode, team visibility in `on`, and shadow mode records would-be denials. Preserve the existing parent-bound sandbox fallback behavior. - Narrow the CHANGELOG claim to active-user session item routes. Workspace-wide session lists, bulk export, and WebSocket authorization remain follow-up work; no route currently makes a session private or team-owned. Follow-up to merged PR #2118. Issue: https://linear.app/colemurray/issue/COL-198/teams-pr-5-control-plane-session-route-requirement-on-every-sessionsid ## Validation On this branch, freshly based on `main`: - `npm run build -w @open-inspect/shared`: passed. - `npm test -w @open-inspect/control-plane -- --run src/routes/session-children.test.ts`: 9 passed. - `npm run test:integration -w @open-inspect/control-plane -- --run test/integration/session-access-routes.test.ts test/integration/child-session-ops.test.ts test/integration/spawn-children.test.ts`: 61 passed. - `npm run typecheck`, `npm run lint:fix`, `npm run format:check`, and `npm run lint:sql-portability`: passed. The red integration case before the fix showed a visible parent listing a private child: ``` AssertionError: expected [ …(3) ] to deeply equal [ …(2) ] ``` The first full integration run also exposed a sandbox-fallback fixture failure, fixed by retaining the bound sandbox principal path. Full control-plane unit and integration suites passed before this one commit was cherry-picked onto updated `main` (5,301 unit tests; 1,456 integration tests, 1 skipped). Co-authored-by: Cole Murray <2492022+ColeMurray@users.noreply.github.com> --- CHANGELOG.md | 8 +++-- .../src/routes/session-children.test.ts | 18 +++++++++++ .../src/routes/session-children.ts | 12 +++++++- .../integration/session-access-routes.test.ts | 30 +++++++++++++++++++ 4 files changed, 64 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2cb76e94c9..875c8d5b21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,9 +6,11 @@ New features, integrations, and notable improvements to Open-Inspect — newest ### Added -`TEAMS_ENFORCEMENT` controls session route access (`off`, `shadow` by default, or `on`). Private -sessions remain restricted in every mode; with enforcement on, session deletion requires the owner, -a team lead, or a workspace administrator. +`TEAMS_ENFORCEMENT` controls active-user session item routes (`/sessions/:id` and its subpaths) +using the persisted session row (`off`, `shadow` by default, or `on`). On those routes, private +visibility applies in every mode; team visibility and the delete ownership rule apply when `on`. +Workspace-wide session lists, bulk export, and WebSocket authorization follow in subsequent changes. +No route can make a session private or team-owned before those changes land. ## September 28, 2026 diff --git a/packages/control-plane/src/routes/session-children.test.ts b/packages/control-plane/src/routes/session-children.test.ts index be5c344599..8038d73c47 100644 --- a/packages/control-plane/src/routes/session-children.test.ts +++ b/packages/control-plane/src/routes/session-children.test.ts @@ -1,4 +1,6 @@ import { afterEach, describe, expect, it, vi } from "vitest"; +import { evaluateSessionAdmission } from "../authorization/session-admission"; +import type * as SessionAdmissionModule from "../authorization/session-admission"; import { SessionIndexStore } from "../db/session-index"; import { resolveSandboxSettings } from "../session/integration-settings-resolution"; import type { SessionRuntimeClient } from "../session/runtime-client"; @@ -13,6 +15,11 @@ vi.mock("../session/integration-settings-resolution", () => ({ resolveSandboxSettings: vi.fn(), })); +vi.mock("../authorization/session-admission", async (importOriginal) => ({ + ...(await importOriginal()), + evaluateSessionAdmission: vi.fn(), +})); + function routeMatch(path: string, pattern: string): { id: string; childId: string } { const match = path.match(routePathPattern(pattern)); if (!match?.groups?.id || !match.groups.childId) throw new Error("Expected route match"); @@ -53,6 +60,10 @@ describe("handleListChildren", () => { afterEach(() => vi.restoreAllMocks()); it("projects viewer-neutral child summaries through the shared schema", async () => { + vi.mocked(evaluateSessionAdmission).mockResolvedValue({ + kind: "allowed", + legacyPermission: "sessions.read", + }); vi.spyOn(SessionIndexStore.prototype, "listByParent").mockResolvedValue([ { id: "child", @@ -119,6 +130,13 @@ describe("handleListChildren", () => { }, ], }); + expect(evaluateSessionAdmission).toHaveBeenCalledWith( + expect.anything(), + expect.anything(), + "child", + "read", + null + ); }); }); diff --git a/packages/control-plane/src/routes/session-children.ts b/packages/control-plane/src/routes/session-children.ts index 1ebf1234c3..276332e416 100644 --- a/packages/control-plane/src/routes/session-children.ts +++ b/packages/control-plane/src/routes/session-children.ts @@ -11,6 +11,7 @@ import { import { DEFAULT_MAX_CONCURRENT_CHILD_SESSIONS } from "@open-inspect/shared/types/integrations"; import { childSessionListResponseSchema } from "@open-inspect/shared/types/sessions"; import { SessionIndexStore, type ChildAdmissionLease } from "../db/session-index"; +import { evaluateSessionAdmission } from "../authorization/session-admission"; import { createLogger } from "../logger"; import { SessionInternalPaths } from "../session/contracts"; import { resolveSandboxSettings } from "../session/integration-settings-resolution"; @@ -41,8 +42,17 @@ export async function handleListChildren( const sessionStore = new SessionIndexStore(ctx.db); const children = await sessionStore.listByParent(parentId); + const visible = []; + for (const child of children) { + if ( + ctx.principal?.kind === "sandbox" || + (await evaluateSessionAdmission(ctx, env, child.id, "read", null)).kind === "allowed" + ) { + visible.push(child); + } + } - return json(childSessionListResponseSchema.parse({ children })); + return json(childSessionListResponseSchema.parse({ children: visible })); } export async function handleGetChild( diff --git a/packages/control-plane/test/integration/session-access-routes.test.ts b/packages/control-plane/test/integration/session-access-routes.test.ts index 3465bc861a..d3cdc3f94b 100644 --- a/packages/control-plane/test/integration/session-access-routes.test.ts +++ b/packages/control-plane/test/integration/session-access-routes.test.ts @@ -346,6 +346,36 @@ describe("HTTP session access by enforcement mode", () => { ).toMatchObject([{ team_id: child.team.id }, { team_id: child.team.id }]); }); + it("lists only children visible in the selected enforcement mode", async () => { + const parent = await initSession({ userId: CREATOR }); + await waitForSandboxStatus(parent.stub, "failed"); + const workspace = await initSession({ userId: CREATOR }); + await waitForSandboxStatus(workspace.stub, "failed"); + const team = await session("team"); + const hidden = await session("private"); + for (const childId of [workspace.sessionName, team.sessionName, hidden.sessionName]) { + await env.DB.prepare("UPDATE sessions SET parent_session_id = ? WHERE id = ?") + .bind(parent.sessionName, childId) + .run(); + } + + for (const mode of ["off", "shadow", "on"] as const) { + const response = await fetchMode(`/sessions/${parent.sessionName}/children`, mode, { + as: { userId: MEMBER, role: "member" }, + }); + expect(response.status).toBe(200); + const body = (await response.json()) as { children: { id: string }[] }; + expect(body.children.map((child) => child.id).sort()).toEqual( + (mode === "on" ? [workspace.sessionName] : [workspace.sessionName, team.sessionName]).sort() + ); + } + expect( + (await auditRows("authorization.request_allowed")).filter( + (row) => row.reason_code === "shadow_denied:batch" + ) + ).toHaveLength(1); + }); + it("audits both private reads when an Owner accesses a private child", async () => { const parent = await session("private"); const child = await session("private"); From 0bf39eb9280198c75e9a95423a31914c51c76eb0 Mon Sep 17 00:00:00 2001 From: Rahul Sethuram Date: Tue, 29 Sep 2026 08:49:20 +0400 Subject: [PATCH 10/13] fix(control-plane): keep resumed sandbox when terminal token expired (#2073) A terminal-enabled Daytona session that stays stopped for more than 24 hours loses its workspace when it resumes. On resume the stored terminal token has expired, so the lifecycle manager deletes the resumed sandbox and spawns a fresh one from the repository. Daytona has no snapshots, so that stopped sandbox held the only copy of the working tree, including uncommitted changes and installed dependencies. A missing token has the same effect. That covers a create-time terminal preview failure and a stored token that no longer decrypts. The same condition on a retained-state recovery (`resume_retained`) puts the session on a recovery hold that retrying cannot clear. This change resumes the sandbox without terminal access instead: the terminal URL and token are not published, the same result as a create-time preview failure. Because those recoveries now get past the resume, it also stops a failed boot from deleting a retained source: a boot of the provider object that a retained recovery point names is held instead, whether it came from that recovery or from a later ordinary resume of the same object. On main that deletion already happens whenever the token is valid. ## Mechanism - `storeTtyd` mints the terminal JWT at spawn with a 24-hour expiry (`packages/control-plane/src/sandbox/lifecycle/manager.ts:97`, `:2377-2395`). The token is signed with the sandbox auth token, which the ttyd proxy verifies (`packages/sandbox-runtime/src/sandbox_runtime/ttyd_proxy/server.ts:2-5`). The control plane stores only a hash of that token (`manager.ts:710`), so it cannot mint a new terminal token for a resumed sandbox. - `resumeSandbox` sets `replaceForTerminalCredential` when the provider returns a terminal URL and the stored token is missing or expired (`manager.ts:1439`). For an ordinary resume it then calls `doSpawn(previousGeneration)` (`manager.ts:1482-1490`). `doSpawn` begins with `stopPriorProviderSandbox`, which stops the resumed provider object with intent `destroy` (`manager.ts:787`, `:1580-1622`). Daytona implements that intent as `deleteSandbox` (`packages/control-plane/src/sandbox/providers/daytona-provider.ts:222-224`) and declares `supportsSnapshots: false` (`daytona-provider.ts:86-92`), so nothing is kept. - Daytona resumes stopped and archived sandboxes (`daytona-provider.ts:188-195`) well past 24 hours, and it is the only persistent-resume provider that returns a terminal URL from `resumeSandbox` (`daytona-provider.ts:199-211`). - On a retained-state recovery the same condition calls `holdFailedRecovery` (`manager.ts:1440-1443`) after the provider has already started the sandbox. The "restore" recovery action resumes the same sandbox again (`packages/control-plane/src/session/sandbox-shutdown.ts:455-476`) and finds the same expired token, so the session stays on hold. - When a retained recovery's resume does commit, `recordProviderStartup` moves the shutdown state from `restoring` to `running` (`sandbox-shutdown.ts:267`) before the bridge reconnects, so the generic watchdogs run. If that boot then fails, the connect watchdog (`manager.ts:1814-1817`), the boot budget (`manager.ts:1951-1963`) and the fatal-runtime-error path (`manager.ts:2115`) stop the sandbox with intent `destroy`, and the next prompt spawns a fresh one. The missing/expired-token case never reached this on main because it stopped at the hold above. The replacement was added in #1988 so that a resumed terminal would never show a URL with an unusable token. This change still guarantees that: without a valid token, neither URL nor token is stored, and the UI hides the terminal. The difference is that the workspace is kept and the terminal is dropped. ## Fix - When the terminal credential is unusable, `resumeSandbox` logs `sandbox.resume_terminal_credential_unavailable` (now at warn), commits the resume with `ttyd: null`, and completes normally. This applies to both ordinary resumes and retained-state recovery. A valid token is handled as before: the refreshed URL is stored next to the preserved token. A later fresh spawn or snapshot restore mints a new token as it does today. - When a boot of the provider object a retained receipt names fails (connect timeout, boot budget or fatal runtime error), `SandboxShutdownCoordinator.holdFailedRetainedBoot` records that object as the provider handle and holds the session with `holdFailedRecovery`, as a failed retained resume already is. Recording the handle matters when the failure lands before the resume commits: an ordinary resume reserves its generation without one, and without it "restore saved" is not offered. It does not fence the row, send the runtime `shutdown`, or stop the sandbox. Skipping the fence matters: `fenceSandboxGeneration` clears the sandbox auth token hash (`packages/control-plane/src/session/sandbox-repository.ts:280-284`), and the next resume of the same sandbox needs it. "Restore saved" then preserve-stops the source and resumes it, and "discard" remains available. The receipt stays in the shutdown record across later generations, and a persistent resume keeps the object id, so this also covers a later ordinary resume of that object, which is still the only copy of the workspace. Generations running any other object fail as before. Not covered: a prompt or typing event can still replace a retained source stuck in `connecting`. `spawnSandbox` treats a row older than the 240 s spawn timeout as dead and `doSpawn` stops the prior object with intent `destroy`. That only happens between the connect-watchdog deadline, which is the same 240 s, and the alarm firing. Main has the same replacement for a `failed` row left by a resume error and for a `ready` row with no bridge. That is replacement admission for persistent-resume providers, which is left to a separate change. A terminal link issued earlier for the same sandbox is not revoked. The ttyd proxy checks only the JWT signature, against the sandbox's unchanged auth token, and its expiry, so the link keeps working until the JWT expires (24 hours after minting) or its signed preview URL does. Main already behaves this way for a running sandbox whose stored token no longer decrypts (the access response omits the terminal, `packages/control-plane/src/session/sandbox-access-reader.ts:39`, `:65`) and for the retained-recovery hold above, which leaves the resumed sandbox running. On main an ordinary resume ended such a link only by deleting the sandbox. ## Verification - `manager.test.ts`: - "keeps a resumed sandbox without terminal access when its terminal token is missing/expired" replaces the two cases that asserted replacement. On unmodified main both fail at `expect(stopSandbox).not.toHaveBeenCalled()` ("called 1 times"). - "keeps resuming a sandbox whose initial terminal preview could not be issued" replaces the test that asserted replacement after a create-time preview failure. On main it fails at `expect(createSandbox).toHaveBeenCalledOnce()` ("got 2 times"). - "resumes retained saved state without terminal access when its terminal token expired" is new. On main it fails at `expect(shutdown.holdFailedRecovery).not.toHaveBeenCalled()`. - "holds a resumed retained source after a connect timeout / fatal runtime error / boot budget instead of deleting it" is new. It runs the real shutdown coordinator with an expired terminal token and has no bridge report ready. Without the hold, the connect-timeout and boot-budget cases fail at `expect(stopSandbox).not.toHaveBeenCalled()` (called with `intent: "destroy"`), and the fatal case fails because `terminateFailedSandbox` reports that it took the sandbox down. - "keeps restore available after a later ordinary resume of the retained source fails mid-resume" is new. A fatal report lands while the resume call is still pending, and the test checks that "restore saved" is offered and resumes the same object. - All pass with the fix. "refreshes terminal URL after resume without replacing its token" (valid token) is unchanged and passes. - A throwaway test (not committed) with the real coordinator and `DaytonaSandboxProvider` over a mock REST client. On main, a retained recovery with a valid token calls `deleteSandbox` after a connect timeout, a boot budget or a fatal runtime error, and so does one with the terminal disabled after a connect timeout; the next prompt then creates a fresh sandbox. With this change none of those cases, and none of the missing/expired-token ones, calls `deleteSandbox`, and "restore saved" resumes the same sandbox. - `npm test -w @open-inspect/control-plane`: 329 files, 5,313 tests passed. `npm run test:integration -w @open-inspect/control-plane`: 120 files, 1,432 tests passed, 1 skipped. - `npm run typecheck -w @open-inspect/control-plane`, ESLint and Prettier on the changed files: clean. - Not run: a live Daytona session resumed after the token expired, or a live resume whose boot failed. ## Summary by CodeRabbit * **Bug Fixes** * Saved workspaces are retained when a resume attempt fails, including after connection timeouts, boot-budget expiry, and fatal runtime errors. Recovery can be retried without replacing the retained sandbox. * Sandboxes now resume in place when terminal credentials are missing or expired, without triggering a replacement. Terminal access remains unavailable until valid credentials are provided. --- .../src/sandbox/lifecycle/manager.test.ts | 231 +++++++++++++++--- .../src/sandbox/lifecycle/manager.ts | 76 +++--- .../src/sandbox/lifecycle/test-helpers.ts | 1 + .../src/session/sandbox-shutdown.ts | 17 ++ 4 files changed, 265 insertions(+), 60 deletions(-) diff --git a/packages/control-plane/src/sandbox/lifecycle/manager.test.ts b/packages/control-plane/src/sandbox/lifecycle/manager.test.ts index 565e2482dc..e09203ce83 100644 --- a/packages/control-plane/src/sandbox/lifecycle/manager.test.ts +++ b/packages/control-plane/src/sandbox/lifecycle/manager.test.ts @@ -202,9 +202,10 @@ async function expectEarlyBridgeStartup(kind: ProviderStartupKind): Promise { function fixture( provider = createMockProvider(), - sandbox = createMockSandbox({ status: "stopped" }) + sandbox = createMockSandbox({ status: "stopped" }), + session = createMockSession() ) { - const storage = createMockStorage(createMockSession(), sandbox); + const storage = createMockStorage(session, sandbox); const sockets = createMockWebSocketManager(); const shutdown = { ...createUnmanagedShutdown(), @@ -336,6 +337,130 @@ describe("final graceful shutdown lifecycle integration", () => { expect(f.provider.createSandbox).not.toHaveBeenCalled(); }); + it.each(["connect timeout", "fatal runtime error", "boot budget"] as const)( + "holds a resumed retained source after a %s instead of deleting it", + async (failure) => { + vi.useFakeTimers(); + try { + const resumeSandbox = vi.fn(async () => ({ + success: true as const, + providerObjectId: "retained-source", + lifetime: noLifetime(), + ttydUrl: "https://terminal.test/resumed", + })); + const stopSandbox = vi.fn(async () => ({ success: true })); + const f = fixture( + createMockProvider({ + resumeSandbox, + stopSandbox, + capabilities: { supportsPersistentResume: true, supportsExplicitStop: true }, + }), + createMockSandbox({ + status: "stopped", + modal_object_id: "retained-source", + // The repository clears this on resume; the mock does not. + last_heartbeat: null, + ttyd_token: await mintJwt( + { exp: Math.floor(Date.now() / 1000) - 1 }, + "sandbox-auth-token" + ), + }), + createMockSession({ sandbox_settings: JSON.stringify({ terminalEnabled: true }) }) + ); + const saved = withSavedState(f, "retained"); + + await f.manager.spawnSandbox(); + expect(saved.read().phase).toBe("running"); + const row = f.storage.getSandbox()!; + if (failure === "fatal runtime error") { + row.last_heartbeat = Date.now(); + expect(await f.manager.terminateFailedSandbox("runtime failed")).toBe(false); + } else { + vi.advanceTimersByTime( + failure === "connect timeout" + ? DEFAULT_LIFECYCLE_CONFIG.connectingTimeout.timeoutMs + 1 + : DEFAULT_LIFECYCLE_CONFIG.bootBudget.timeoutMs + 1 + ); + if (failure === "boot budget") row.last_heartbeat = Date.now(); + expect(await f.manager.handleShutdownAlarm()).toBe("continue"); + await f.manager.handleAlarm(); + } + + // Neither deleted nor fenced: the source is the only copy of the workspace. + expect(stopSandbox).not.toHaveBeenCalled(); + expect(row).toMatchObject({ + status: "failed", + modal_object_id: "retained-source", + fenced: 0, + }); + expect(saved.read()).toMatchObject({ + phase: "unknown", + receipt: { kind: "retained", artifactId: "retained-source" }, + }); + await f.manager.spawnSandbox(); + expect(f.provider.createSandbox).not.toHaveBeenCalled(); + + await saved.shutdown.recover("restore_saved"); + expect(stopSandbox).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining({ providerObjectId: "retained-source", intent: "preserve" }) + ); + await f.manager.spawnSandbox(); + expect(resumeSandbox).toHaveBeenCalledTimes(2); + expect(saved.read().phase).toBe("running"); + expect(f.provider.createSandbox).not.toHaveBeenCalled(); + } finally { + vi.useRealTimers(); + } + } + ); + + it("keeps restore available after a later ordinary resume of the retained source fails mid-resume", async () => { + let finishResume!: (result: ResumeResult) => void; + const resumed = { + success: true as const, + providerObjectId: "retained-source", + lifetime: noLifetime(), + }; + const resumeSandbox = vi + .fn>() + .mockResolvedValueOnce(resumed) + .mockReturnValueOnce(new Promise((resolve) => (finishResume = resolve))) + .mockResolvedValue(resumed); + const stopSandbox = vi.fn(async () => ({ success: true })); + const f = fixture( + createMockProvider({ + resumeSandbox, + stopSandbox, + capabilities: { supportsPersistentResume: true, supportsExplicitStop: true }, + }), + createMockSandbox({ status: "stopped", modal_object_id: "retained-source" }) + ); + const saved = withSavedState(f, "retained"); + await f.manager.spawnSandbox(); + // A heartbeat timeout preserve-stops it; the shutdown record stays running. + const row = f.storage.getSandbox()!; + row.status = "stopped"; + + const ordinaryResume = f.manager.spawnSandbox(); + await vi.waitFor(() => expect(resumeSandbox).toHaveBeenCalledTimes(2)); + row.last_heartbeat = Date.now(); + expect(await f.manager.terminateFailedSandbox("runtime failed")).toBe(false); + finishResume(resumed); + await ordinaryResume; + + expect(stopSandbox).not.toHaveBeenCalled(); + expect(saved.read()).toMatchObject({ phase: "unknown", providerObjectId: "retained-source" }); + expect(saved.shutdown.snapshot()?.availableRecoveryActions).toContain("restore_saved"); + await saved.shutdown.recover("restore_saved"); + expect(stopSandbox).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining({ providerObjectId: "retained-source", intent: "preserve" }) + ); + await f.manager.spawnSandbox(); + expect(resumeSandbox).toHaveBeenCalledTimes(3); + expect(saved.read().phase).toBe("running"); + expect(f.provider.createSandbox).not.toHaveBeenCalled(); + }); + it("allows only explicit retry of an ambiguous snapshot restore from a retired source", async () => { const restoreFromSnapshot = vi .fn>() @@ -1981,7 +2106,7 @@ describe("SandboxLifecycleManager", () => { }); it.each(["missing", "expired"] as const)( - "replaces a resumable sandbox when its terminal token is %s", + "keeps a resumed sandbox without terminal access when its terminal token is %s", async (credentialState) => { const ttydToken = credentialState === "expired" @@ -2003,7 +2128,6 @@ describe("SandboxLifecycleManager", () => { providerObjectId: "replacement-provider-obj", createdAt: Date.now(), lifetime: noLifetime(), - ttydUrl: "https://terminal.test/replacement", })); const resumeSandbox = vi.fn(async () => ({ success: true as const, @@ -2032,22 +2156,19 @@ describe("SandboxLifecycleManager", () => { await manager.spawnSandbox(); - expect(resumeSandbox).toHaveBeenCalled(); - expect(stopSandbox).toHaveBeenCalledWith( - expect.objectContaining({ - providerObjectId: "old-provider-obj", - reason: "respawn", - }) - ); - expect(createSandbox).toHaveBeenCalledWith( - expect.objectContaining({ sandboxSettings: { terminalEnabled: true } }) - ); - expect(sandbox.ttyd_token).not.toBeNull(); - expect(sandbox.ttyd_token).not.toBe(ttydToken); + // The stopped sandbox is the only copy of the workspace; losing the + // terminal must not delete it. + expect(resumeSandbox).toHaveBeenCalledOnce(); + expect(stopSandbox).not.toHaveBeenCalled(); + expect(createSandbox).not.toHaveBeenCalled(); + expect(sandbox.modal_object_id).toBe("old-provider-obj"); + expect(sandbox.status).toBe("connecting"); + expect(sandbox.ttyd_url).toBeNull(); + expect(sandbox.ttyd_token).toBeNull(); } ); - it("replaces a resumed sandbox after its initial terminal preview could not be issued", async () => { + it("keeps resuming a sandbox whose initial terminal preview could not be issued", async () => { const sandbox = createMockSandbox({ status: "pending", created_at: Date.now() - 60_000, @@ -2059,28 +2180,24 @@ describe("SandboxLifecycleManager", () => { createMockSession({ sandbox_settings: JSON.stringify({ terminalEnabled: true }) }), sandbox ); - let createCount = 0; - const createSandbox = vi.fn(async (config: CreateSandboxConfig) => { - createCount++; - return { - sandboxId: config.sandboxId, - providerObjectId: createCount === 1 ? "initial-provider-obj" : "replacement-provider-obj", - createdAt: Date.now(), - lifetime: noLifetime(), - ...(createCount === 2 ? { ttydUrl: "https://terminal.test/replacement" } : {}), - }; - }); + const createSandbox = vi.fn(async (config: CreateSandboxConfig) => ({ + sandboxId: config.sandboxId, + providerObjectId: "initial-provider-obj", + createdAt: Date.now(), + lifetime: noLifetime(), + })); const resumeSandbox = vi.fn(async () => ({ success: true as const, providerObjectId: "initial-provider-obj", lifetime: noLifetime(), ttydUrl: "https://terminal.test/resumed", })); + const stopSandbox = vi.fn(async () => ({ success: true })); const provider = createMockProvider({ capabilities: { supportsExplicitStop: true, supportsPersistentResume: true }, createSandbox, resumeSandbox, - stopSandbox: vi.fn(async () => ({ success: true })), + stopSandbox, }); const manager = new SandboxLifecycleManager( provider, @@ -2100,9 +2217,59 @@ describe("SandboxLifecycleManager", () => { await manager.spawnSandbox(); expect(resumeSandbox).toHaveBeenCalledOnce(); - expect(createSandbox).toHaveBeenCalledTimes(2); - expect(sandbox.ttyd_url).toBe("https://terminal.test/replacement"); - expect(sandbox.ttyd_token).not.toBeNull(); + expect(createSandbox).toHaveBeenCalledOnce(); + expect(stopSandbox).not.toHaveBeenCalled(); + expect(sandbox.modal_object_id).toBe("initial-provider-obj"); + expect(sandbox.ttyd_url).toBeNull(); + }); + + it("resumes retained saved state without terminal access when its terminal token expired", async () => { + const sandbox = createMockSandbox({ + status: "stopped", + modal_object_id: "retained-source", + ttyd_url: null, + ttyd_token: await mintJwt({ exp: Math.floor(Date.now() / 1000) - 1 }, "sandbox-auth-token"), + }); + const storage = createMockStorage( + createMockSession({ sandbox_settings: JSON.stringify({ terminalEnabled: true }) }), + sandbox + ); + const provider = createMockProvider({ + capabilities: { supportsExplicitStop: true, supportsPersistentResume: true }, + resumeSandbox: vi.fn(async () => ({ + success: true as const, + providerObjectId: "retained-source", + lifetime: noLifetime(), + ttydUrl: "https://terminal.test/resumed", + })), + }); + const shutdown = createUnmanagedShutdown(); + shutdown.startupDecision.mockReturnValue({ + kind: "resume_retained", + providerObjectId: "retained-source", + runtimeVersion: COMPATIBLE_RUNTIME_VERSION, + }); + const manager = new SandboxLifecycleManager( + provider, + storage, + storage, + createMockBroadcaster(), + createMockWebSocketManager(false), + createMockAlarmScheduler(), + createMockIdGenerator(), + shutdown, + createTestConfig() + ); + + await manager.spawnSandbox(); + + // Retrying recovery cannot renew the credential, so holding here would + // leave the saved workspace unrecoverable. + expect(shutdown.holdFailedRecovery).not.toHaveBeenCalled(); + expect(shutdown.recordProviderStartup).toHaveBeenCalledOnce(); + expect(provider.createSandbox).not.toHaveBeenCalled(); + expect(sandbox.modal_object_id).toBe("retained-source"); + expect(sandbox.ttyd_url).toBeNull(); }); it("does not carry a predecessor's runtime version onto a replacement's snapshot", async () => { diff --git a/packages/control-plane/src/sandbox/lifecycle/manager.ts b/packages/control-plane/src/sandbox/lifecycle/manager.ts index 4156c6bd65..285f7c30b7 100644 --- a/packages/control-plane/src/sandbox/lifecycle/manager.ts +++ b/packages/control-plane/src/sandbox/lifecycle/manager.ts @@ -152,6 +152,8 @@ export interface SandboxShutdownLifecycle { ): Promise; /** Decides startup without exposing the coordinator's persisted receipt representation. */ startupDecision(): SandboxStartupDecision; + /** Holds a failed boot of the retained source, which deleting would lose; false for other objects. */ + holdFailedRetainedBoot(error: string, generation: SandboxGeneration): boolean; /** Converts a failed or interrupted saved-state startup into a durable safety hold. */ holdFailedRecovery(error: string, generation?: SandboxGeneration): void; /** Records runtime protocol support; does not itself grant lifecycle command readiness. */ @@ -1493,10 +1495,15 @@ export class SandboxLifecycleManager ? await this.storage.getSandboxAccessSecret("ttyd") : null; const validTtydToken = ttydToken && isJwtUnexpired(ttydToken) ? ttydToken : null; - const replaceForTerminalCredential = Boolean(result.ttydUrl && !validTtydToken); - if (replaceForTerminalCredential && restoringSavedState) { - this.shutdown.holdFailedRecovery("Terminal credential is missing or expired", generation); - return; + if (result.ttydUrl && !validTtydToken) { + // Terminal tokens are signed with the sandbox auth token, which is kept + // only as a hash, so an expired or missing one cannot be renewed. The + // resumed sandbox holds the workspace; keep it without terminal access. + this.log.warn("Terminal credential unavailable; resuming without terminal access", { + event: "sandbox.resume_terminal_credential_unavailable", + provider_object_id: finalProviderObjectId, + reason: ttydToken ? "invalid_or_expired" : "missing", + }); } let completed: boolean; try { @@ -1507,12 +1514,7 @@ export class SandboxLifecycleManager ? { url: result.codeServerUrl, password: result.codeServerPassword } : null, vnc: result.vncAccess ?? null, - ttyd: validTtydToken - ? { - url: replaceForTerminalCredential ? null : (result.ttydUrl ?? null), - token: validTtydToken, - } - : null, + ttyd: validTtydToken ? { url: result.ttydUrl ?? null, token: validTtydToken } : null, tunnelUrls: result.tunnelUrls ?? null, }); } catch (error) { @@ -1536,16 +1538,6 @@ export class SandboxLifecycleManager await this.shutdown.recordProviderStartup(generation, result.lifetime); startupClaimed = true; - if (replaceForTerminalCredential) { - this.log.info("Terminal credential unavailable; replacing resumed sandbox", { - event: "sandbox.resume_terminal_credential_unavailable", - provider_object_id: finalProviderObjectId, - reason: ttydToken ? "invalid_or_expired" : "missing", - }); - await this.doSpawn(previousGeneration); - return; - } - if (!this.broadcastSandboxDashboardUrl(finalProviderObjectId)) { this.broadcaster.broadcast({ type: "sandbox_access_changed" }); } @@ -1845,6 +1837,20 @@ export class SandboxLifecycleManager } } + /** + * Hold a failed boot of the retained source instead of destroying it. That + * sandbox is the saved workspace itself, and a fence would revoke the + * credential its next resume needs. Like a failed retained resume, it waits + * for an explicit recovery, which retires the source before resuming it + * again. Resolves false for any other generation. + */ + private holdFailedRetainedBoot(sandbox: SandboxRow, reason: string): boolean { + return this.shutdown.holdFailedRetainedBoot(reason, { + sandboxId: sandbox.modal_sandbox_id, + createdAt: sandbox.created_at, + }); + } + /** * Give up on a generation whose bridge never arrived. The row is failed and * the breaker charged before the provider stop, so a prompt landing mid-stop @@ -1862,7 +1868,11 @@ export class SandboxLifecycleManager this.storage.updateSandboxStatus("failed"); this.recordSpawnFailure(ctx.now, ctx.sandbox.created_at); this.clearSandboxAccessState(); - if (this.canStopProviderSandbox()) { + const held = this.holdFailedRetainedBoot( + ctx.sandbox, + "Sandbox failed to connect within the allowed time" + ); + if (!held && this.canStopProviderSandbox()) { // Fenced before the stop: a bridge arriving while the stop is in // flight is refused at the door instead of self-healing into a // container being killed. Where the provider cannot be stopped the @@ -1879,7 +1889,9 @@ export class SandboxLifecycleManager } this.broadcaster.broadcast({ type: "sandbox_status", status: "failed" }); this.reportSandboxError( - "Sandbox failed to connect within the allowed time. It will be retried on your next message." + held + ? "Sandbox failed to connect within the allowed time." + : "Sandbox failed to connect within the allowed time. It will be retried on your next message." ); return "sandbox_failed"; } @@ -1988,9 +2000,10 @@ export class SandboxLifecycleManager * stopped; only then is the row failed. The failure is published and * persisted before the provider stop yields, and the spawn guard is held * across it, so a prompt arriving mid-stop neither waits to learn the boot - * died nor reserves a replacement that inherits this failure. Returns the - * failure text so the alarm handler can fail the pending prompt with the - * same words. + * died nor reserves a replacement that inherits this failure. A boot of the + * retained source is held instead, and its runtime and sandbox are left for + * the recovery. Returns the failure text so the alarm handler can fail the + * pending prompt with the same words. */ private async failBootBudget(elapsedMs: number, ctx: AlarmContext): Promise { const bootPhase = parseStoredSandboxBootPhase(ctx.sandbox.boot_phase); @@ -2004,13 +2017,17 @@ export class SandboxLifecycleManager elapsed_ms: elapsedMs, timeout_ms: this.config.bootBudget.timeoutMs, }); - this.wsManager.sendToSandbox({ type: "shutdown" }); - this.storage.fenceSandboxGeneration(); + const held = this.holdFailedRetainedBoot(ctx.sandbox, reason); + if (!held) { + this.wsManager.sendToSandbox({ type: "shutdown" }); + this.storage.fenceSandboxGeneration(); + } this.storage.updateSandboxStatus("failed"); this.recordSpawnFailure(ctx.now, ctx.sandbox.created_at); this.clearSandboxAccessState(); this.broadcaster.broadcast({ type: "sandbox_status", status: "failed" }); this.reportSandboxError(reason); + if (held) return { kind: "boot_budget_exceeded", reason }; this.wsManager.detachSandboxWebSocket(1000, "Boot budget exceeded"); if (this.canStopProviderSandbox()) { this.isTerminatingSandbox = true; @@ -2131,7 +2148,8 @@ export class SandboxLifecycleManager * provider allows. Resolves true only when this call took the sandbox down, * which is the caller's cue to re-evaluate the queue. Serving executions * remain fenced by preservation until explicit recovery; only failed boots - * may automatically get a clean replacement. A row + * may automatically get a clean replacement, and a failed boot of the + * retained source is held instead. A row * that is already dead — including one the connect watchdog failed while * its boot was still running — resolves false: there is nothing to * terminate, and re-driving the queue for it would spawn a replacement for @@ -2159,11 +2177,13 @@ export class SandboxLifecycleManager return ownership === "owned"; } if (!this.isCurrentSandboxState(sandbox)) return false; + const held = this.holdFailedRetainedBoot(sandbox, reason); this.storage.updateSandboxStatus("failed"); this.recordSpawnFailure(Date.now(), sandbox.created_at); this.broadcaster.broadcast({ type: "sandbox_status", status: "failed" }); this.reportSandboxError(reason); this.clearSandboxAccessState(); + if (held) return false; const canStopProvider = this.canStopProviderSandbox(); if (!canStopProvider) this.wsManager.sendToSandbox({ type: "shutdown" }); diff --git a/packages/control-plane/src/sandbox/lifecycle/test-helpers.ts b/packages/control-plane/src/sandbox/lifecycle/test-helpers.ts index 7e5065989d..4094ac152c 100644 --- a/packages/control-plane/src/sandbox/lifecycle/test-helpers.ts +++ b/packages/control-plane/src/sandbox/lifecycle/test-helpers.ts @@ -468,6 +468,7 @@ export function createUnmanagedShutdown() { startupDecision: vi.fn(() => ({ kind: "normal", })), + holdFailedRetainedBoot: vi.fn(() => false), holdFailedRecovery: vi.fn(), runtimeReady: vi.fn(), generationReady: vi.fn(), diff --git a/packages/control-plane/src/session/sandbox-shutdown.ts b/packages/control-plane/src/session/sandbox-shutdown.ts index f76bbc65b3..6bae341944 100644 --- a/packages/control-plane/src/session/sandbox-shutdown.ts +++ b/packages/control-plane/src/session/sandbox-shutdown.ts @@ -429,6 +429,23 @@ export class SandboxShutdownCoordinator { : { kind: "normal" }; } + /** + * Holds a failed boot of the source a retained receipt names, as a failed + * retained resume is held, and records that source so recovery can retire + * and resume it: an ordinary resume reserves its generation without a + * provider handle. Holds nothing and resolves false for any other object. + */ + holdFailedRetainedBoot(error: string, generation: SandboxGeneration): boolean { + const state = this.deps.store.read(); + const source = state?.receipt?.kind === "retained" ? state.receipt.artifactId : null; + if (!state || !source || this.deps.sandbox.getSandbox()?.modal_object_id !== source) + return false; + if (this.current(state) && this.matches(state, generation)) + this.deps.store.write({ ...state, providerObjectId: source }); + this.holdFailedRecovery(error, generation); + return true; + } + holdFailedRecovery(error: string, generation?: SandboxGeneration): void { const state = this.deps.store.read(); const row = this.deps.sandbox.getSandbox(); From 2b51245f1a6fa3eac773961fe87470ec6dd1c702 Mon Sep 17 00:00:00 2001 From: Rahul Sethuram Date: Tue, 29 Sep 2026 08:51:38 +0400 Subject: [PATCH 11/13] fix(control-plane): project token totals for steps that land after a stop (#2081) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a turn is stopped, the session's token totals in D1 (`sessions.input_tokens` and the other token columns added in #2067) can stay below the session's actual step usage. `/sessions/export` and `/analytics/runs` then report lower totals than the session's own `step_usage` rows, which the export's `usage` collection reads from the session (`/internal/trace-export`). The difference lasts until another turn settles. If the session cost limit caused the stop and another prompt is queued, no turn settles while the budget stays exhausted, so the lower totals remain. ## Mechanism - A stop ends the turn in the control plane before the sandbox acts on it. `ExecutionStopCoordinator.prepare` marks the processing message failed (`packages/control-plane/src/session/execution-stop-coordinator.ts:43-56`). `deliver` then sends `stop` to the sandbox and calls `reconcileAfterExecution(false)` (`:67-72`). - With nothing queued, the session moves to `failed` and `syncSessionMetrics` writes the token totals at that point. - With a prompt queued, the session stays `active`. That is not a settled status, so nothing is written (`packages/control-plane/src/session/session-status-service.ts:227-233`, `:55-73`). - The turn keeps running until the sandbox processes `stop` (`packages/sandbox-runtime/src/sandbox_runtime/bridge.py:630-631`). A `step_finish` already on the socket, or emitted before the cancel, therefore arrives after the stop. If the cancel interrupts a `step_finish` send, the event is buffered again (`packages/sandbox-runtime/src/sandbox_runtime/event_forwarder.py:190-201`) and is only replayed on the next bind, so it arrives even later. - `SandboxStreamingEventHandler.handleStep` records every `step_finish` in `step_usage` and cost, whatever state its message is in, so the usage rows include the late step. - The sandbox's own `execution_complete` for the stopped turn finds no processing message. It logs `already_stopped` and does not reconcile (`packages/control-plane/src/session/sandbox-events/execution.handler.ts:67-77`, `:116-122`). The projection only refreshes when a transition settles a turn, so nothing rewrites D1 after the late step. - The step whose cost reaches the session limit triggers a budget stop (`packages/control-plane/src/session/budget-service.ts:121-158`). If a prompt is queued, the session stays `active`, and the queue does not dispatch while `budget_exhausted = 1` (`packages/control-plane/src/session/message-queue.ts:404-406`). No later settle projects the stopped turn at all. ## Fix - `SessionStatusService.refreshMetricsAfterStep(messageId)` re-projects metrics after a `step_finish` unless the step's own message is still `processing`. The step's turn decides this, not the session status: - A step of a stopped or finished turn is projected, even while a queued prompt keeps the session `active`. - A step of the processing turn waits for the next settle, so steps during a running turn add no D1 writes. - In a budget stop, the step that reaches the limit ends its own turn while it is ingested. The refresh after that step already writes the stopped turn's totals, and each late step writes them again. - `handleStep` now submits that refresh through `BackgroundTasks` instead of calling it in its `finally`. The factory still runs synchronously, so the refresh reads state at the same point. The task boundary absorbs and logs a failed refresh, for example `getSession()` throwing on a malformed session row (`packages/control-plane/src/session/session-core-repository.ts:314-317`). A failed refresh can therefore no longer replace the step's result or its usage/budget error. - Metrics writes are last-write-wins, and a late-step refresh can overlap a settle's write. To stop older values from landing after newer ones, only one write is in flight per session, and each write reads the session when it runs. - A request made while a write is in flight marks it stale, and the writer runs one more pass with the current values. - If the in-flight write fails, the queued pass still runs. The first failure is rethrown to the background-task boundary once the writer drains, so it is still logged. - A failed write with nothing newer pending is not retried. - With no write in flight, the first write is still issued synchronously. Unchanged from main, and out of scope here because neither case involves a step landing late: - A turn that ends while a prompt is queued is still projected only by the next settle, not at that boundary. The budget can then hold the queue with no later step landing, either because a limit edit below the running total stopped the turn or because the turn's own `execution_complete` carried the cost report that reached the limit. In that case D1 waits until the limit is raised or the queued prompt is removed. - A stopped turn's own `execution_complete` can still raise `total_cost` through its final cumulative cost report. That cost-only change is not projected until the next settle. No migration. ## Verification - `test/integration/sandbox-events.test.ts`, "projects a budget-stopped turn's steps while a queued prompt waits on the budget": - Setup: a processing turn and a queued prompt under a $1 limit. Events: a step, a step whose cost reaches the limit, a late `step_finish`, then the sandbox's cancelled `execution_complete`. - Checks: the session stays `active` with the budget exhausted and the prompt still pending. D1 already holds the stopped turn's tokens once the limit is reached. D1's token totals end equal to the sums of the usage rows from `/internal/trace-export?include=usage`. - It fails on main: D1 `inputTokens` is 0 once the limit is reached (expected 300). It also fails on the previous head, where every D1 token column is still 0 at the end. - `test/integration/sandbox-events.test.ts`, "projects a step that finishes after a stop settled the session": a step, then `/internal/stop`, then a late `step_finish` and the sandbox's cancelled `execution_complete`. On main it fails at the final `toMatchObject`: D1 has `inputTokens: 100` (expected 350), and the other token kinds are likewise at the first step's values. - `sandbox-events/processor.test.ts`: - "keeps a recorded step's result when refreshing its metrics fails": with the refresh throwing, the step resolves and the failure is recorded at the task boundary. When the refresh was called in `finally`, the step rejected with the refresh error. - "reports the usage error rather than a failed metrics refresh": when the usage write and the refresh both throw, the step rejects with the usage error. When the refresh was called in `finally`, the refresh error replaced it. - `session-status-service.test.ts`: - "defers a step of the processing turn but projects one whose turn has ended": on an `active` session, a step of the processing turn writes nothing, and a step of a stopped turn writes the current totals. - "writes usage that lands during a metrics write after it, never beside it": holds the first `updateMetrics` open and checks that later refreshes are not issued alongside it. After the first write resolves, exactly one more write follows, with the newer totals. - "still writes usage that landed during a metrics write that failed": rejects the first write after a refresh marked it stale. It checks that a second write follows with the newer totals and that the failure is reported once. Without the per-pass error handling, it fails with `updateMetrics` called 1 time instead of 2. - "does not retry a failed metrics write when nothing newer is pending": checks that a single failed write is reported and not retried, and that the next refresh writes normally. - Rebased onto main at 700f9145. `npm test -w @open-inspect/control-plane`: 329 files, 5,314 tests passed. `vitest.integration.config.ts` (full): 120 files, 1,434 tests passed, 1 skipped. `npm run typecheck -w @open-inspect/control-plane`, plus ESLint and Prettier on the changed files: clean. ## Summary by CodeRabbit * **Bug Fixes** * Session usage metrics now include usage from steps that finish after a stop has settled the session, without allowing a later failed terminal event to change the settled state. * Overlapping metric updates are serialized and use the latest available totals. If an update fails, newer pending usage can still be processed; failed updates aren’t retried unless a newer update is pending or another refresh is requested. --- .../control-plane/src/session/components.ts | 3 +- .../session/sandbox-events/processor.test.ts | 45 ++++- .../sandbox-events/streaming.handler.ts | 38 +++-- .../session/session-status-service.test.ts | 77 +++++++++ .../src/session/session-status-service.ts | 92 +++++++--- .../test/integration/sandbox-events.test.ts | 157 ++++++++++++++++++ 6 files changed, 376 insertions(+), 36 deletions(-) diff --git a/packages/control-plane/src/session/components.ts b/packages/control-plane/src/session/components.ts index 1723e34300..2df0e0178c 100644 --- a/packages/control-plane/src/session/components.ts +++ b/packages/control-plane/src/session/components.ts @@ -552,7 +552,8 @@ export function createSessionRuntime(platform: SessionPlatform, env: Env): Sessi messenger, updateLastActivity, budgetService, - usageRepository + usageRepository, + (messageId) => statusService.refreshMetricsAfterStep(messageId) ); const artifactEventHandler = new SandboxArtifactEventHandler( artifactRepository, diff --git a/packages/control-plane/src/session/sandbox-events/processor.test.ts b/packages/control-plane/src/session/sandbox-events/processor.test.ts index 8b13d50523..fe64a7fdac 100644 --- a/packages/control-plane/src/session/sandbox-events/processor.test.ts +++ b/packages/control-plane/src/session/sandbox-events/processor.test.ts @@ -120,6 +120,7 @@ function createProcessor( deliverTransition: vi.fn(async () => {}), }; const usageRepository = { recordStepUsage: vi.fn() }; + const refreshMetricsAfterStep = vi.fn((_messageId: string | null) => {}); // The real family composition, mirroring components.ts, so the suite keeps // pinning end-to-end processSandboxEvent behavior across the split. @@ -135,7 +136,8 @@ function createProcessor( messenger, updateLastActivity, budgetService as unknown as SessionBudgetService, - persistedUsage ?? (usageRepository as unknown as UsageRepository) + persistedUsage ?? (usageRepository as unknown as UsageRepository), + refreshMetricsAfterStep ), new SandboxArtifactEventHandler( artifactRepository, @@ -203,6 +205,7 @@ function createProcessor( log, budgetService, usageRepository, + refreshMetricsAfterStep, }; } @@ -515,6 +518,46 @@ describe("SessionSandboxEventProcessor", () => { } ); + it("keeps a recorded step's result when refreshing its metrics fails", async () => { + const h = createProcessor(); + const refreshError = new Error("Malformed persisted session row"); + h.refreshMetricsAfterStep.mockImplementation(() => { + throw refreshError; + }); + + await expect( + h.processor.processSandboxEvent({ + type: "step_finish", + messageId: "msg-1", + sandboxId: "sb-1", + timestamp: 1000, + tokens: { input: 10 }, + }) + ).resolves.toBeUndefined(); + expect(h.backgroundTasks.failures).toEqual([refreshError]); + }); + + it("reports the usage error rather than a failed metrics refresh", async () => { + const h = createProcessor(); + const persistenceError = new Error("usage write failed"); + h.usageRepository.recordStepUsage.mockImplementationOnce(() => { + throw persistenceError; + }); + h.refreshMetricsAfterStep.mockImplementation(() => { + throw new Error("Malformed persisted session row"); + }); + + await expect( + h.processor.processSandboxEvent({ + type: "step_finish", + messageId: "msg-1", + sandboxId: "sb-1", + timestamp: 1000, + cost: 0.25, + }) + ).rejects.toBe(persistenceError); + }); + it("records unavailable cost tracking for positive-token steps without cost", async () => { const h = createProcessor(); const event: SandboxEvent = { diff --git a/packages/control-plane/src/session/sandbox-events/streaming.handler.ts b/packages/control-plane/src/session/sandbox-events/streaming.handler.ts index d3631913e2..3c9ea4b0db 100644 --- a/packages/control-plane/src/session/sandbox-events/streaming.handler.ts +++ b/packages/control-plane/src/session/sandbox-events/streaming.handler.ts @@ -13,8 +13,10 @@ import { persistSandboxEvent, type SandboxEventContext } from "./context"; * execution (tokens, steps, tool activity, compaction). Every event here is * broadcast to clients; the ones with a durable representation also record * to the timeline (steps renew activity, accumulate cost, and persist usage). Nothing - * here transitions session state. Also owns the timeline-observer path - * (`recordTimelineEvent`) for events that persist and broadcast unchanged. + * here transitions session state; a step whose turn has already ended + * refreshes the metrics projection itself, as no settle for that turn is still + * to come. Also owns the timeline-observer path (`recordTimelineEvent`) for + * events that persist and broadcast unchanged. */ export class SandboxStreamingEventHandler { constructor( @@ -24,7 +26,8 @@ export class SandboxStreamingEventHandler { private readonly messenger: SessionMessenger, private readonly updateLastActivity: (timestamp: number) => void, private readonly budgetService: SessionBudgetService, - private readonly usageRepository: UsageRepository + private readonly usageRepository: UsageRepository, + private readonly refreshMetricsAfterStep: (messageId: string | null) => void ) {} handleToken(event: Extract, context: SandboxEventContext): void { @@ -56,19 +59,28 @@ export class SandboxStreamingEventHandler { this.updateLastActivity(context.now); this.messenger.broadcast({ type: "sandbox_event", event }); if (event.type === "step_finish") { - let persistenceFailure: { error: unknown } | null = null; try { - this.usageRepository.recordStepUsage(event, context.messageId, context.now); - } catch (error) { - persistenceFailure = { error }; - } - try { - await this.budgetService.ingestStepFinish(event, context.messageId, context.now); - } catch (error) { + let persistenceFailure: { error: unknown } | null = null; + try { + this.usageRepository.recordStepUsage(event, context.messageId, context.now); + } catch (error) { + persistenceFailure = { error }; + } + try { + await this.budgetService.ingestStepFinish(event, context.messageId, context.now); + } catch (error) { + if (persistenceFailure) throw persistenceFailure.error; + throw error; + } if (persistenceFailure) throw persistenceFailure.error; - throw error; + } finally { + // Submitted so a failed refresh is logged at the task boundary rather + // than replacing the step's own outcome. + this.backgroundTasks.submit(async () => this.refreshMetricsAfterStep(context.messageId), { + name: "session_index.refresh_step_metrics", + context: { message_id: context.messageId }, + }); } - if (persistenceFailure) throw persistenceFailure.error; } } diff --git a/packages/control-plane/src/session/session-status-service.test.ts b/packages/control-plane/src/session/session-status-service.test.ts index dfb177885b..f80ab09ee4 100644 --- a/packages/control-plane/src/session/session-status-service.test.ts +++ b/packages/control-plane/src/session/session-status-service.test.ts @@ -63,6 +63,7 @@ function harness(options: { session?: SessionRow | null } = {}) { updateSessionStatus: vi.fn(), getPendingOrProcessingCount: vi.fn(() => 0), getLatestTerminalMessage: vi.fn(() => null as MessageRow | null), + getMessageStatus: vi.fn((_messageId: string): MessageRow["status"] | null => "failed"), getMessageCount: vi.fn(() => 3), getActiveDurationMs: vi.fn(() => 4500), }; @@ -292,6 +293,82 @@ describe("SessionStatusService.transition", () => { expect(h.sessionIndex.updateMetrics).not.toHaveBeenCalled(); }); + it("defers a step of the processing turn but projects one whose turn has ended", () => { + // A budget stop leaves a queued prompt that keeps the session active. + const h = harness({ session: createSession({ status: "active" }) }); + h.repository.getMessageStatus.mockReturnValueOnce("processing"); + + h.service.refreshMetricsAfterStep("msg-running"); + expect(h.sessionIndex.updateMetrics).not.toHaveBeenCalled(); + + h.service.refreshMetricsAfterStep("msg-stopped"); + expect(h.sessionIndex.updateMetrics).toHaveBeenCalledWith( + "public-session-1", + expect.objectContaining({ inputTokens: 1200 }) + ); + }); + + it("writes usage that lands during a metrics write after it, never beside it", async () => { + const h = harness({ session: createSession({ status: "failed" }) }); + let releaseFirstWrite!: () => void; + h.sessionIndex.updateMetrics.mockImplementationOnce( + () => new Promise((resolve) => (releaseFirstWrite = () => resolve(true))) + ); + + expect(await h.service.transition("failed")).toBe(false); + h.usageRepository.getSessionTotals.mockReturnValue(createUsageTotals({ inputTokens: 1500 })); + h.service.refreshMetricsAfterStep("msg-1"); + h.service.refreshMetricsAfterStep("msg-1"); + + expect(h.sessionIndex.updateMetrics).toHaveBeenCalledTimes(1); + releaseFirstWrite(); + await h.backgroundTasks.settle(); + + expect(h.sessionIndex.updateMetrics).toHaveBeenCalledTimes(2); + expect(h.sessionIndex.updateMetrics).toHaveBeenLastCalledWith( + "public-session-1", + expect.objectContaining({ inputTokens: 1500 }) + ); + }); + + it("still writes usage that landed during a metrics write that failed", async () => { + const h = harness({ session: createSession({ status: "failed" }) }); + const error = new Error("d1 down"); + let failFirstWrite!: () => void; + h.sessionIndex.updateMetrics.mockImplementationOnce( + () => new Promise((_resolve, reject) => (failFirstWrite = () => reject(error))) + ); + + expect(await h.service.transition("failed")).toBe(false); + h.usageRepository.getSessionTotals.mockReturnValue(createUsageTotals({ inputTokens: 1500 })); + h.service.refreshMetricsAfterStep("msg-1"); + failFirstWrite(); + await h.backgroundTasks.settle(); + + expect(h.sessionIndex.updateMetrics).toHaveBeenCalledTimes(2); + expect(h.sessionIndex.updateMetrics).toHaveBeenLastCalledWith( + "public-session-1", + expect.objectContaining({ inputTokens: 1500 }) + ); + expect(h.backgroundTasks.failures).toEqual([error]); + }); + + it("does not retry a failed metrics write when nothing newer is pending", async () => { + const h = harness({ session: createSession({ status: "failed" }) }); + const error = new Error("d1 down"); + h.sessionIndex.updateMetrics.mockRejectedValueOnce(error); + + expect(await h.service.transition("failed")).toBe(false); + await h.backgroundTasks.settle(); + + expect(h.sessionIndex.updateMetrics).toHaveBeenCalledTimes(1); + expect(h.backgroundTasks.failures).toEqual([error]); + + h.service.refreshMetricsAfterStep("msg-1"); + await h.backgroundTasks.settle(); + expect(h.sessionIndex.updateMetrics).toHaveBeenCalledTimes(2); + }); + it("logs index sync failures without throwing", async () => { const h = harness({ session: createSession({ status: "created" }) }); h.statusProjection.project.mockRejectedValue(new Error("d1 down")); diff --git a/packages/control-plane/src/session/session-status-service.ts b/packages/control-plane/src/session/session-status-service.ts index 191ca7546f..7544d7f4f9 100644 --- a/packages/control-plane/src/session/session-status-service.ts +++ b/packages/control-plane/src/session/session-status-service.ts @@ -27,6 +27,11 @@ import { isSessionPromptable, isTurnSettled } from "@open-inspect/shared/types/s type SessionIndexProjections = Pick; export class SessionStatusService { + /** A metrics write is in flight; later requests fold into its next pass. */ + private metricsSyncInFlight = false; + /** State changed after the in-flight write read it. */ + private metricsSyncStale = false; + constructor( private readonly backgroundTasks: BackgroundTasks, private readonly log: Logger, @@ -170,6 +175,24 @@ export class SessionStatusService { return true; } + /** + * Re-project metrics for a step whose turn is no longer processing. A stop + * ends the turn before the sandbox has seen the stop, so a step already in + * flight lands afterwards, and the sandbox's own terminal for that turn then + * settles nothing. The turn decides, not the session: a budget stop leaves a + * queued prompt that keeps the session `active` but cannot dispatch, so no + * later settle would cover the step. A step of the processing turn waits + * for the next settle. + */ + refreshMetricsAfterStep(messageId: string | null): void { + if (messageId !== null && this.messageRepository.getMessageStatus(messageId) === "processing") { + return; + } + const session = this.repository.getSession(); + if (!session) return; + this.syncSessionMetrics(this.getPublicSessionId(session)); + } + private async projectTransition( session: SessionRow, publicSessionId: string, @@ -338,30 +361,34 @@ export class SessionStatusService { }); } + /** + * Writes are last-write-wins, so at most one is in flight, and each reads + * the session when it runs: a request made during a write only marks it + * stale, and the write goes round again with the newer state instead of + * racing it to D1. A failed pass still goes round when a newer request + * arrived during it; the first failure is reported once the writer drains. + */ private syncSessionMetrics(sessionId: string): void { - const session = this.repository.getSession(); - if (!session) return; - - const messageCount = this.messageRepository.getMessageCount(); - const activeDurationMs = this.messageRepository.getActiveDurationMs(); - const artifacts = this.artifactRepository.listArtifacts(); - const prCount = artifacts.filter((a) => a.type === "pr").length; + if (this.metricsSyncInFlight) { + this.metricsSyncStale = true; + return; + } + if (!this.repository.getSession()) return; + this.metricsSyncInFlight = true; this.backgroundTasks.submit( - () => { - // The index keeps aggregate-friendly zeros; "unknown" lives in the usage rows. - const tokens = this.usageRepository.getSessionTotals(); - return this.sessionIndex.updateMetrics(sessionId, { - totalCost: session.total_cost ?? 0, - activeDurationMs, - messageCount, - prCount, - inputTokens: tokens.inputTokens ?? 0, - outputTokens: tokens.outputTokens ?? 0, - reasoningTokens: tokens.reasoningTokens ?? 0, - cacheReadTokens: tokens.cacheReadTokens ?? 0, - cacheWriteTokens: tokens.cacheWriteTokens ?? 0, - }); + async () => { + let failure: { error: unknown } | null = null; + do { + this.metricsSyncStale = false; + try { + await this.projectSessionMetrics(sessionId); + } catch (error) { + failure ??= { error }; + } + } while (this.metricsSyncStale); + this.metricsSyncInFlight = false; + if (failure) throw failure.error; }, { name: "session_index.update_metrics", @@ -369,4 +396,27 @@ export class SessionStatusService { } ); } + + private async projectSessionMetrics(sessionId: string): Promise { + const session = this.repository.getSession(); + if (!session) return; + + const messageCount = this.messageRepository.getMessageCount(); + const activeDurationMs = this.messageRepository.getActiveDurationMs(); + const artifacts = this.artifactRepository.listArtifacts(); + const prCount = artifacts.filter((a) => a.type === "pr").length; + // The index keeps aggregate-friendly zeros; "unknown" lives in the usage rows. + const tokens = this.usageRepository.getSessionTotals(); + await this.sessionIndex.updateMetrics(sessionId, { + totalCost: session.total_cost ?? 0, + activeDurationMs, + messageCount, + prCount, + inputTokens: tokens.inputTokens ?? 0, + outputTokens: tokens.outputTokens ?? 0, + reasoningTokens: tokens.reasoningTokens ?? 0, + cacheReadTokens: tokens.cacheReadTokens ?? 0, + cacheWriteTokens: tokens.cacheWriteTokens ?? 0, + }); + } } diff --git a/packages/control-plane/test/integration/sandbox-events.test.ts b/packages/control-plane/test/integration/sandbox-events.test.ts index b9214db57e..d4329fad5a 100644 --- a/packages/control-plane/test/integration/sandbox-events.test.ts +++ b/packages/control-plane/test/integration/sandbox-events.test.ts @@ -8,6 +8,7 @@ import { queryDO, seedMessage, seedSandboxAuth, + waitForSandboxStatus, } from "./helpers"; import { runInSessionDO } from "./session-do-access"; @@ -452,6 +453,162 @@ describe("POST /internal/sandbox-event", () => { }); }); + it("projects a step that finishes after a stop settled the session", async () => { + const { stub, sessionName } = await initSession(); + const participants = await queryDO<{ id: string }>( + stub, + "SELECT id FROM participants WHERE user_id = 'user-1'" + ); + const msgId = "msg-late-step"; + await seedMessage(stub, { + id: msgId, + authorId: participants[0].id, + content: "Test prompt", + source: "web", + status: "processing", + createdAt: Date.now() - 1000, + startedAt: Date.now() - 500, + }); + const postEvent = (event: Record) => + stub.fetch("http://internal/internal/sandbox-event", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ sandboxId: "sb-1", messageId: msgId, ...event }), + }); + const stepFinish = (stepId: string, input: number) => + postEvent({ + type: "step_finish", + stepId, + timestamp: Date.now() / 1000, + tokens: { input, output: 40, reasoning: 5, cache: { read: 800, write: 60 } }, + }); + const index = new SessionIndexStore(env.DB); + + expect((await stepFinish("step-1", 100)).status).toBe(200); + // Stop settles the session before the sandbox has seen the stop command. + expect((await stub.fetch("http://internal/internal/stop", { method: "POST" })).status).toBe( + 200 + ); + await vi.waitFor(async () => { + expect(await index.get(sessionName)).toMatchObject({ status: "failed", inputTokens: 100 }); + }); + + // A step already in flight lands after the settle; its terminal is then a no-op. + expect((await stepFinish("step-2", 250)).status).toBe(200); + const res = await postEvent({ + type: "execution_complete", + success: false, + error: "Task was cancelled", + timestamp: Date.now() / 1000, + }); + expect(res.status).toBe(200); + + await vi.waitFor(async () => { + expect(await index.get(sessionName)).toMatchObject({ + inputTokens: 350, + outputTokens: 80, + reasoningTokens: 10, + cacheReadTokens: 1600, + cacheWriteTokens: 120, + }); + }); + }); + + it("projects a budget-stopped turn's steps while a queued prompt waits on the budget", async () => { + const { stub, sessionName } = await initSession({ sandboxSettings: { maxSessionCostUsd: 1 } }); + await waitForSandboxStatus(stub, "failed"); + const [{ id: authorId }] = await queryDO<{ id: string }>( + stub, + "SELECT id FROM participants WHERE user_id = 'user-1'" + ); + const stoppedId = "msg-budget-stopped"; + const queuedId = "msg-budget-queued"; + await seedMessage(stub, { + id: stoppedId, + authorId, + content: "Running prompt", + source: "web", + status: "processing", + createdAt: Date.now() - 1000, + startedAt: Date.now() - 500, + }); + await seedMessage(stub, { + id: queuedId, + authorId, + content: "Queued prompt", + source: "web", + status: "pending", + createdAt: Date.now() - 900, + }); + await queryDO(stub, "UPDATE session SET status = 'active'"); + const postEvent = (event: Record) => + stub.fetch("http://internal/internal/sandbox-event", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ sandboxId: "sb-1", messageId: stoppedId, ...event }), + }); + const stepFinish = (stepId: string, input: number, messageCostUsd: number) => + postEvent({ + type: "step_finish", + stepId, + timestamp: Date.now() / 1000, + tokens: { input, output: 40, reasoning: 5, cache: { read: 800, write: 60 } }, + messageCostUsd, + }); + const index = new SessionIndexStore(env.DB); + + expect((await stepFinish("step-1", 100, 0.4)).status).toBe(200); + // This step reaches the limit: the budget stops the turn, and the queued + // prompt keeps the session active but cannot dispatch. The stopped turn is + // projected anyway, since no settle is coming. + expect((await stepFinish("step-2", 200, 1.2)).status).toBe(200); + await vi.waitFor(async () => { + expect(await index.get(sessionName)).toMatchObject({ inputTokens: 300 }); + }); + // A step already in flight lands after the stop, then the sandbox's own terminal. + expect((await stepFinish("step-3", 250, 1.5)).status).toBe(200); + const terminal = await postEvent({ + type: "execution_complete", + success: false, + error: "Task was cancelled", + timestamp: Date.now() / 1000, + }); + expect(terminal.status).toBe(200); + + expect(await queryDO(stub, "SELECT status, budget_exhausted FROM session")).toEqual([ + { status: "active", budget_exhausted: 1 }, + ]); + expect(await queryDO(stub, "SELECT id, status FROM messages ORDER BY created_at")).toEqual([ + { id: stoppedId, status: "failed" }, + { id: queuedId, status: "pending" }, + ]); + const tokenKinds = [ + "inputTokens", + "outputTokens", + "reasoningTokens", + "cacheReadTokens", + "cacheWriteTokens", + ] as const; + const exported = await stub.fetch("http://internal/internal/trace-export?include=usage"); + expect(exported.status).toBe(200); + const { trace } = await exported.json<{ + trace: { usage: Array> }; + }>(); + const usageTotals = Object.fromEntries( + tokenKinds.map((kind) => [kind, trace.usage.reduce((sum, row) => sum + (row[kind] ?? 0), 0)]) + ); + expect(usageTotals).toEqual({ + inputTokens: 550, + outputTokens: 120, + reasoningTokens: 15, + cacheReadTokens: 2400, + cacheWriteTokens: 180, + }); + await vi.waitFor(async () => { + expect(await index.get(sessionName)).toMatchObject(usageTotals); + }); + }); + it("execution_complete with success=false marks message as failed", async () => { const { stub } = await initSession(); From e471cff4f6dd5b695e0f7bad82ff16aed2b55775 Mon Sep 17 00:00:00 2001 From: Rahul Sethuram Date: Tue, 29 Sep 2026 08:57:58 +0400 Subject: [PATCH 12/13] fix(web): fail trace download on in-stream error records (#2119) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem The single-session export route (`GET /sessions/:id/export`) always answers `200 application/x-ndjson` once the session exists. Failures that happen while producing the record are encoded inside the stream (`packages/control-plane/src/routes/session-export.ts`): - a trace read timeout or runtime failure becomes a `{"type":"session_error", ...}` line instead of the `session` line (`streamExport`, ~line 248–252, via `sessionErrorLine`); - any other exception while streaming becomes a `{"type":"error"}` line (~line 259). The "Download trace" action in `packages/web/src/components/session-right-sidebar.tsx` (`downloadTrace`, ~line 101) only checks `response.ok`, so in both cases it saves an error-only `session-.ndjson` file and shows no failure toast. ## Fix After reading the body, check its NDJSON records; if any record has type `session_error` or `error`, throw so the existing `catch` shows "Failed to download trace" and no file is downloaded. Successful traces are downloaded unchanged. (A body that is not valid NDJSON also ends in the failure toast now, because `JSON.parse` throws into the same `catch`.) ## Reproduction / tests Added `it.each(["session_error", "error"])("reports a %s export record as a failed download")` to `session-right-sidebar.test.tsx`: mocks a 200 NDJSON response containing only that record and expects the failure toast with no anchor click / object URL. Both cases fail on current `main` and pass with the fix. Ran in `packages/web`: `npm run typecheck`, `npx vitest run` (full web suite), plus prettier and eslint on the two touched files. ## Summary by CodeRabbit * **Bug Fixes** * Trace downloads now show an error notification and prevent downloading when the response contains an error or cannot be parsed. --- .../components/session-right-sidebar.test.tsx | 29 +++++++++++++++++++ .../src/components/session-right-sidebar.tsx | 11 ++++++- 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/packages/web/src/components/session-right-sidebar.test.tsx b/packages/web/src/components/session-right-sidebar.test.tsx index 6e3afc673b..eb40ca570c 100644 --- a/packages/web/src/components/session-right-sidebar.test.tsx +++ b/packages/web/src/components/session-right-sidebar.test.tsx @@ -132,6 +132,35 @@ describe("SessionRightSidebar", () => { expect(click.mock.instances[0]).toHaveProperty("href", "blob:session-trace"); }); + it.each(["session_error", "error"])( + "reports a %s export record as a failed download", + async (type) => { + vi.mocked(browserApiFetch).mockResolvedValueOnce( + new Response(`{"schemaVersion":1,"type":"${type}"}\n`, { + headers: { "Content-Type": "application/x-ndjson" }, + }) + ); + const click = vi.spyOn(HTMLAnchorElement.prototype, "click").mockImplementation(() => {}); + render( + + ); + + fireEvent.click(screen.getByRole("button", { name: "Download trace" })); + await waitFor(() => expect(toast.error).toHaveBeenCalledWith("Failed to download trace")); + expect(click).not.toHaveBeenCalled(); + expect(URL.createObjectURL).not.toHaveBeenCalled(); + } + ); + it("aborts a trace body that stalls after headers and re-enables the download button", async () => { vi.useFakeTimers(); let fetchSignal: AbortSignal | undefined; diff --git a/packages/web/src/components/session-right-sidebar.tsx b/packages/web/src/components/session-right-sidebar.tsx index 79201f3bd7..3507285e69 100644 --- a/packages/web/src/components/session-right-sidebar.tsx +++ b/packages/web/src/components/session-right-sidebar.tsx @@ -110,7 +110,16 @@ export function SessionRightSidebarContent({ ); if (!response.ok) throw new Error("Trace export failed"); - const url = URL.createObjectURL(await response.blob()); + const blob = await response.blob(); + // Trace read and stream failures arrive as NDJSON records inside a 200 response. + const failed = (await blob.text()).split("\n").some((line) => { + if (!line) return false; + const { type } = JSON.parse(line) as { type?: unknown }; + return type === "session_error" || type === "error"; + }); + if (failed) throw new Error("Trace export failed"); + + const url = URL.createObjectURL(blob); const link = document.createElement("a"); link.href = url; link.download = `session-${sessionId}.ndjson`; From ff8daa38a83abefc02c5593673f3ad8004f74b26 Mon Sep 17 00:00:00 2001 From: Rahul Sethuram Date: Tue, 29 Sep 2026 10:09:23 +0400 Subject: [PATCH 13/13] fix(sandbox-runtime): keep Bun's fetch timeout on non-generation calls --- .../src/sandbox_runtime/plugins/codex-auth-plugin.js | 5 +++-- packages/sandbox-runtime/tests/codex-auth-plugin.test.mjs | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/sandbox-runtime/src/sandbox_runtime/plugins/codex-auth-plugin.js b/packages/sandbox-runtime/src/sandbox_runtime/plugins/codex-auth-plugin.js index 1de685585b..ba9c24877b 100644 --- a/packages/sandbox-runtime/src/sandbox_runtime/plugins/codex-auth-plugin.js +++ b/packages/sandbox-runtime/src/sandbox_runtime/plugins/codex-auth-plugin.js @@ -626,8 +626,9 @@ export const CodexAuthProxy = async (input) => { const { accessToken, accountId } = await ensureAccessToken(getAuth, setAuth); proxied.headers.set("authorization", `Bearer ${accessToken}`); if (accountId) proxied.headers.set("ChatGPT-Account-Id", accountId); - // Let OpenCode's stream timeout handle stalls instead of Bun's fetch idle timer. - return fetch(proxied, { timeout: false }); + // No generation stream here, so OpenCode's stream timeout does not + // bound it: Bun's fetch timeout stays in force. + return fetch(proxied); } const { headers, inherited, method, body, signal } = await normalizeRequest( diff --git a/packages/sandbox-runtime/tests/codex-auth-plugin.test.mjs b/packages/sandbox-runtime/tests/codex-auth-plugin.test.mjs index 5abd5909ed..aa72f984f3 100644 --- a/packages/sandbox-runtime/tests/codex-auth-plugin.test.mjs +++ b/packages/sandbox-runtime/tests/codex-auth-plugin.test.mjs @@ -856,6 +856,7 @@ test("dispatches a non-generation Request without reshaping it", async () => { assert.equal(call.request.headers.get("authorization"), "Bearer cp-access"); const sent = new Uint8Array(await call.request.arrayBuffer()); assert.deepEqual([...sent], [...body], "the body bytes are untouched"); + assert.equal(call.init.timeout, undefined, "Bun's fetch timeout stays in force"); }); test("cancels a stalled streaming body when the caller aborts", async () => {