From c2f0335aec1198d82e3dd7c2dca45995ac442aae Mon Sep 17 00:00:00 2001 From: Cole Murray Date: Tue, 22 Sep 2026 01:01:33 -0700 Subject: [PATCH 01/22] feat(sandbox): add distinct Modal VM backend identity --- .env.example | 1 + docs/GETTING_STARTED.md | 3 +- docs/MODAL_DOCKER.md | 70 +++ .../2026-09-22-modal-vm-backend-identities.md | 490 ++++++++++++++++++ .../src/image-builds/modal-adapter.ts | 10 +- .../control-plane/src/image-builds/model.ts | 1 + .../control-plane/src/image-builds/planner.ts | 1 + .../src/image-builds/provider-factory.ts | 3 +- .../control-plane/src/image-builds/types.ts | 2 + .../src/routes/image-builds.trigger.test.ts | 2 + packages/control-plane/src/sandbox/client.ts | 67 ++- .../src/sandbox/lifecycle/manager.test.ts | 43 +- .../src/sandbox/lifecycle/manager.ts | 36 +- .../lifecycle/rejected-allocation.test.ts | 58 +++ .../src/sandbox/lifecycle/test-helpers.ts | 15 + .../src/sandbox/provider-factory.ts | 12 +- .../control-plane/src/sandbox/provider.ts | 16 + .../sandbox/providers/modal-backends.test.ts | 195 +++++++ .../sandbox/providers/modal-provider.test.ts | 74 +-- .../src/sandbox/providers/modal-provider.ts | 111 +++- .../control-plane/src/session/components.ts | 2 +- .../src/session/sandbox-access.ts | 3 +- .../src/session/sandbox-repository.test.ts | 37 ++ .../src/session/sandbox-repository.ts | 18 + .../integration/modal-backend-builds.test.ts | 110 ++++ packages/modal-infra/README.md | 2 + packages/modal-infra/deploy.py | 94 +++- packages/modal-infra/src/images/base.py | 34 +- .../modal-infra/src/sandbox/build_session.py | 86 ++- .../modal-infra/src/sandbox/launch_policy.py | 119 +++++ packages/modal-infra/src/sandbox/manager.py | 224 ++++++-- packages/modal-infra/src/web_api.py | 99 +++- .../tests/test_build_sandbox_lifecycle.py | 96 +++- packages/modal-infra/tests/test_deploy.py | 121 +++++ .../modal-infra/tests/test_docker_launch.py | 113 ++++ .../modal-infra/tests/test_sandbox_launch.py | 338 ++++++++++++ .../tests/test_sandbox_resources.py | 13 +- .../tests/test_snapshot_timeout.py | 61 ++- .../tests/test_web_api_build_sandbox.py | 70 ++- .../tests/test_web_api_create_sandbox.py | 72 +++ .../sandbox-images/install/docker-daemon.json | 19 + packages/sandbox-images/install/docker.sh | 37 ++ .../src/sandbox_images/bundle.py | 17 + packages/sandbox-images/tests/test_bundle.py | 61 +++ packages/sandbox-images/toolchain.json | 22 + .../sandbox-images/verify/docker_smoke.py | 153 ++++++ .../src/sandbox_runtime/bridge.py | 4 + .../src/sandbox_runtime/constants.py | 4 + .../src/sandbox_runtime/docker_control.py | 79 +++ .../src/sandbox_runtime/docker_service.py | 193 +++++++ .../src/sandbox_runtime/entrypoint.py | 2 + .../image_build_context_start.py | 4 +- .../src/sandbox_runtime/runtime_config.py | 5 + .../src/sandbox_runtime/supervisor.py | 114 +++- .../tests/test_docker_control.py | 46 ++ .../tests/test_docker_service.py | 217 ++++++++ .../tests/test_runtime_config.py | 12 + .../tests/test_supervisor_lifecycle.py | 236 +++++++++ packages/shared/src/types/integrations.ts | 2 + packages/web/src/lib/sandbox-provider.test.ts | 8 +- packages/web/src/lib/sandbox-provider.ts | 9 +- terraform/environments/production/locals.tf | 2 +- terraform/environments/production/modal.tf | 1 + .../production/terraform.tfvars.example | 7 +- .../tests/modal_backends.tftest.hcl | 70 +++ .../environments/production/variables.tf | 26 +- terraform/modules/modal-app/main.tf | 16 +- .../modal-app/tests/backends.tftest.hcl | 28 + terraform/modules/modal-app/variables.tf | 6 + 69 files changed, 4115 insertions(+), 207 deletions(-) create mode 100644 docs/MODAL_DOCKER.md create mode 100644 docs/plans/2026-09-22-modal-vm-backend-identities.md create mode 100644 packages/control-plane/src/sandbox/lifecycle/rejected-allocation.test.ts create mode 100644 packages/control-plane/src/sandbox/providers/modal-backends.test.ts create mode 100644 packages/control-plane/test/integration/modal-backend-builds.test.ts create mode 100644 packages/modal-infra/src/sandbox/launch_policy.py create mode 100644 packages/modal-infra/tests/test_docker_launch.py create mode 100644 packages/sandbox-images/install/docker-daemon.json create mode 100644 packages/sandbox-images/install/docker.sh create mode 100644 packages/sandbox-images/verify/docker_smoke.py create mode 100644 packages/sandbox-runtime/src/sandbox_runtime/docker_control.py create mode 100644 packages/sandbox-runtime/src/sandbox_runtime/docker_service.py create mode 100644 packages/sandbox-runtime/tests/test_docker_control.py create mode 100644 packages/sandbox-runtime/tests/test_docker_service.py create mode 100644 terraform/environments/production/tests/modal_backends.tftest.hcl create mode 100644 terraform/modules/modal-app/tests/backends.tftest.hcl diff --git a/.env.example b/.env.example index b5643654ae..5007a3e1db 100644 --- a/.env.example +++ b/.env.example @@ -157,6 +157,7 @@ GITLAB_NAMESPACE= # --------------------------------------------------------------------------- SANDBOX_PROVIDER=modal +# Set modal-vm for deployment-wide Docker-capable VMs; see docs/MODAL_DOCKER.md. # Modal. The API secret is shared with the Modal deployment for HMAC-signed # endpoint calls; the workspace and environment build the endpoint URLs. diff --git a/docs/GETTING_STARTED.md b/docs/GETTING_STARTED.md index ddd3110340..1737f5a8a7 100644 --- a/docs/GETTING_STARTED.md +++ b/docs/GETTING_STARTED.md @@ -149,7 +149,8 @@ Create an R2 API Token: ### Modal -> Only required when `sandbox_provider = "modal"`. +> Only required when `sandbox_provider` is `modal` or `modal-vm`. Select `modal-vm` for +> Docker-capable VMs; see [Modal VM setup](MODAL_DOCKER.md). 1. Go to [Modal Settings](https://modal.com/settings) 2. **Create a new API token**: Settings -> API Tokens -> New Token diff --git a/docs/MODAL_DOCKER.md b/docs/MODAL_DOCKER.md new file mode 100644 index 0000000000..91a81d03f3 --- /dev/null +++ b/docs/MODAL_DOCKER.md @@ -0,0 +1,70 @@ +# Modal VM backend + +Open Inspect offers two Modal compute backends: + +| `SANDBOX_PROVIDER` | Runtime | User Docker workloads | +| ------------------ | ----------------------- | --------------------- | +| `modal` (default) | Existing gVisor sandbox | Not enabled | +| `modal-vm` | Modal VM | Included | + +Selection is deployment-wide. There is no Docker checkbox or repository/session override. Mixed use +within one deployment is a follow-up. Both backends share a Modal app, account, credentials, +transport, and implementation; they have separate prepared-image pools. + +## Deployment + +For Terraform, set `sandbox_provider = "modal-vm"`. The existing Modal module builds and verifies +the Docker image before deploying the worker. No extra provisioning/admission flag is required. For +a standalone control plane, set `SANDBOX_PROVIDER=modal-vm` and provision the data plane first: + +```bash +cd packages/modal-infra +BUILD_MODAL_VM_IMAGE=true uv run python deploy.py --build-sandbox-image +BUILD_MODAL_VM_IMAGE=true uv run modal deploy deploy.py +``` + +These commands create billable resources. Use the intended Modal environment and credentials. Never +deploy `src/app.py` directly. VM launches reject old or incompatible API responses rather than +falling back to standard sandboxes. Existing standard clients can still use the standard endpoints. + +## Runtime and resources + +The harness, bridge, workspace, IDE, and desktop run directly on the VM host. Docker is for user +workloads such as PostgreSQL, Redis, and container builds. The runtime supervises the local daemon; +user environment variables cannot enable Docker or redirect its readiness probes. + +Generic `cpuCores` and `memoryMib` size the **outer VM**, not individual containers. Missing/null +values select the offering defaults (currently 2 cores and 4096 MiB). Positive explicit settings +override them. VM image builds use their scope's configured resources. Standard build sizing is +unchanged. These values are product defaults, not claimed Modal minimums. + +## Snapshots and recovery + +VM session snapshots are **destructive**: quiesce Docker, capture the filesystem, then confirm VM +termination. Later work restores into a new VM. Standard Modal snapshots remain non-destructive. A +failed or ambiguous capture/retirement must not be reported as a successful checkpoint. + +Filesystem capture is not process/RAM continuity or an application-consistent database backup. +Containers must use appropriate persistence and restart policies. Live Docker pause/resume is not +provided. Raw daemon logs are truncated after clean preparation before reusable image capture. + +Retried VM launches adopt only an exactly owned allocation and recover its original interactive +credentials. A predecessor must be confirmed terminated before launching a replacement. Build +allocations have deterministic backend/build names and can be recovered after a lost create +response. Returned build handles are persisted for cleanup before backend validation; incompatible +builds never start and cannot publish prepared images. + +## Switching backends + +Changing `SANDBOX_PROVIDER` is an operator cutover, not session migration. Existing sessions and +snapshots may become unusable. Drain/retire active allocations first when feasible, retain +credentials for pending cleanup, and rebuild images under the selected backend. Do not relabel old +artifacts. Rollback to `modal` does not transparently resume VM sessions or prove old VMs have +stopped. + +PR #2007's earlier per-session Docker/variant design was not deployed. Its schema additions and +settings are not part of this implementation, so no variant migration is required. + +Provider-backed canaries must validate Docker startup, build/restore, access after adoption, +snapshot/retirement, and cleanup in the target deployment before production rollout. Unit tests +alone do not prove those provider behaviors. diff --git a/docs/plans/2026-09-22-modal-vm-backend-identities.md b/docs/plans/2026-09-22-modal-vm-backend-identities.md new file mode 100644 index 0000000000..d4a271f668 --- /dev/null +++ b/docs/plans/2026-09-22-modal-vm-backend-identities.md @@ -0,0 +1,490 @@ +# Modal and Modal VM: distinct backend identities + +## Status and scope + +Implementation authorized by the user on 2026-09-22; deployment is not authorized. + +The user confirmed PR #2007 has never been deployed. This implementation starts from current main, +retains the runtime and reviewed Modal fixes, and omits all unpublished variant migrations and UI. +Mixed-backend selection remains a follow-up. + +Prepared against PR #2007, local branch `fix-modal-docker-review-findings`, commit +`5764ad45ba417db73c226154464d5b141740a695`. Paths below are relative to the repository root. Recheck +the PR head, target branch, outstanding review findings, and migration history before coding. + +This decision supersedes the earlier per-session `dockerEnabled` / artifact-variant design. The user +explicitly selected **deployment-wide backend selection now; mixed-backend selection later**. The +user also accepts that changing the deployment backend can orphan existing sessions and their +snapshots, as switching from Modal to another vendor already can. Seamless session migration is not +a requirement. This does not authorize silently reinterpreting incompatible images or forgetting +known resource-cleanup obligations. + +## 1. Decision + +Represent two compute offerings through the existing sandbox-provider contract: + +| Backend ID | Display name | Execution environment | Docker for user workloads | Implementation | +| ---------- | ------------ | ----------------------- | ---------------------------- | -------------------------------------------------- | +| `modal` | Modal | Existing gVisor sandbox | Not enabled by this offering | Shared Modal provider/client | +| `modal-vm` | Modal VM | Modal VM | Included | Same provider/client, immutable offering selection | + +Keep `SANDBOX_PROVIDER` as the deployment-wide selector. Its default remains `modal`. Both +identities use the same Modal account, credentials, HTTP transport, app deployment, image-build +adapter, and shared lifecycle implementation. A separate identity is **not** a separate vendor, +duplicate provider class, mandatory second Modal app, or second Terraform module instance. + +Use existing `provider` fields for the concrete compute backend. Do not rename every existing +provider type/column to `backend`, add a parallel `backend` column, or introduce a provider-family +registry. Documentation should distinguish a compute backend from its vendor where needed. + +The important simplification is removing runtime selection from shared session settings and artifact +selection. Moving `modal-docker.ts` to a differently named shared file is not the solution. + +### Invariants + +1. `modal` retains the existing default launch behavior. `modal-vm` either launches the VM offering + with its Docker-capable image or fails; it never falls back to gVisor. +2. A provider instance has one immutable identity. Calls cannot change it through settings, user + environment variables, image metadata, or a mutable field on a shared client. +3. Prepared images are selected, registered, superseded, and cleaned up under their recorded backend + identity. `modal` images cannot satisfy `modal-vm` lookups, or vice versa. +4. Scheduler, session routes, generic lifecycle policy, and image-build workflow do not know about + Docker admission, VM resource defaults, or Modal runtime flags. +5. CPU and memory configure the outer compute allocation, not individual Docker containers. +6. Existing checkpoint/retirement fencing remains intact. A checkpoint is not proof of retirement. + Preserve [ADR 0004](../adr/0004-sandbox-checkpoint-and-shutdown.md). +7. Backend switching has no continuity guarantee. Existing fail-closed provider checks remain; + accepting orphaned sessions is not permission to report a successful restore from incompatible + state. + +### Explicit non-goals + +- Per-session, repository, environment, child-session, or automation backend overrides. +- Session backend pinning, automatic cross-provider snapshot conversion, or migration UI. +- A general execution-profile, capability-negotiation, allocation-coordinator, or routing framework. +- Untying all compute provisioning from Terraform in this change. +- Separate provider identities for CPU sizes, Docker versions, or routine runtime/image releases. +- Moving the Open Inspect harness into Docker. Runtime, bridge, workspace, and interactive services + remain on the VM host; Docker hosts user workloads. +- Certifying Docker support for other providers or altering their resource/lifecycle semantics. +- Automatically destroying old sandboxes, deleting their data, or deploying this plan. + +## 2. Ownership and contracts + +### 2.1 Provider composition + +Extend the existing backend-name union with `modal-vm`. At the provider factory, construct the same +`ModalSandboxProvider` with an explicit immutable identity of `modal` or `modal-vm`. Require the +factory to pass the identity; tests and direct constructors must do the same. Compute `name` and +capabilities after identity assignment, avoiding field-initializer ordering bugs. + +Keep this two-value type local to the Modal implementation, derived from the existing backend union +where practical. Do not expose `dockerEnabled`, `vmRuntime`, or a configurable launch profile on +`SandboxProvider`. + +The image-build factory maps both identities to `ModalImageBuildAdapter`, passing the exact recorded +identity rather than normalizing both to `modal`. Client creation/cleanup must not require the VM +base image to be currently provisioned; launching a new VM does. This preserves cleanup after a +deployment changes its default backend or removes its VM launch image. + +### 2.2 Modal-private wire contract + +Use an explicit `sandbox_backend: "modal" | "modal-vm"` on Modal HTTP **create, restore, and +create-build** requests. This field is private to the Modal adapter/client and Python API; it is not +a public session-setting field. Both sides validate the finite set and reject unknown values. + +- A missing request field defaults to `modal`, preserving ordinary older callers. +- Remove runtime selection through `sandbox_settings.dockerEnabled`. If an old request explicitly + contains that removed setting, reject it with an actionable error instead of silently ignoring + `true` and launching gVisor. Do not maintain two ways to select the runtime. +- Successful allocation responses carry `sandbox_backend`, derived from the actual launch/adoption + path, not merely echoed from the request. Ownership validation must precede confirmation for an + adopted allocation. Unknown response values are protocol errors. +- A `modal-vm` provider requires an exact `modal-vm` confirmation. Missing, mismatched, or malformed + confirmation is an incompatible deployment response: clean up a known allocation and fail before + returning it to a session or starting an image build. Record build allocation handles before + confirmation so failed cleanup survives restart; binding is not authorization to start work. +- A `modal` provider accepts `modal` confirmation. For rolling upgrades, it may accept a legacy + response lacking the field only if the old `docker_enabled` signal is absent or explicitly false. + It must reject an explicitly VM/Docker response. Never permit this exception for `modal-vm`. +- Emit only `sandbox_backend`. Read legacy `docker_enabled` only when validating an old standard + endpoint response; there is no outbound legacy shim. +- Cleanup failures preserve the resource identifier and error context for retry/reconciliation; do + not claim that the allocation was retired. Do not start a build after failed confirmation. + +This is an application-level launch confirmation, not hardware attestation. VM support is also +validated by the data-plane launch configuration and an explicitly authorized provider canary. The +ordinary runtime-generation compatibility floor remains a separate existing concern. + +### 2.3 Resources + +Retain generic `cpuCores` and `memoryMib` in existing sandbox settings. Remove Docker-specific +freezing, inheritance rules, and positive-resource requirements from the scheduler and session +routes. Existing generic settings resolution/persistence still determines user-requested resources. + +Put the **Modal VM offering defaults** in one Python launch-policy module: preserve the current +product choices of 2 CPU cores and 4096 MiB. These are product defaults, not asserted Modal +minimums. An absent or null resource selects the selected backend's default; a positive explicit +value is honored. Validate finite CPU and positive integer memory at the Modal boundary, including +rejecting booleans. Preserve standard Modal's existing default behavior when resources are +unspecified. + +Use one resource-to-SDK mapping for session create, restore, and VM build allocations. Do not +compose two independently generated dictionaries that overwrite the same `cpu`/`memory` keys. +Deployment verification should import the offering defaults rather than restating them. Tests may +assert explicit expected values; production code has one owner for defaults. + +Image-build resource decision: preserve configured VM build sizing without passing the whole session +settings bag as a runtime selector. Replace the feature-added `ImageBuildPlan.sandboxSettings` with +an optional neutral `resources` value containing only `cpuCores` and `memoryMib`, projected from the +already-resolved scope settings. Reuse those field types, not an extensible execution-profile type. +Only the Modal adapter consumes it in this change. VM builds apply configured values/defaults; +standard builds retain their pre-feature sizing behavior. Do not incidentally change other backends +or invent a new build-resource settings UI. + +No CPU/memory values belong in an artifact identity. This change does not promise that changing an +allocation's resources preserves running processes or makes an application snapshot consistent. + +### 2.4 Images, snapshots, and deployment switching + +Prepared images already have a provider dimension. Use `(scope, provider)` plus existing repository +fingerprints and runtime compatibility checks. Remove `artifactVariant` from the target, planner, +registration, lookup, reconciliation, and workflow contracts. Remove the extra snapshot-variant +projection from the session repository/lifecycle contract. + +Keep recorded provider fields on existing build rows and shutdown receipts. Do not overwrite them +with the deployment's current choice. Recorded image-build cleanup must remain routable when +`SANDBOX_PROVIDER` changes. Existing provider-mismatch recovery checks must continue to reject an +incompatible receipt. No new per-session provider column is required for this scope. + +Changing `SANDBOX_PROVIDER` is an operator cutover, not a migration. Old sessions may be unusable +and must not be described as resumable. This plan neither guarantees every legacy orphan is +identified nor introduces transparent continuity. Operators should drain/retire known active +allocations before switching when feasible; retain credentials/endpoints for outstanding cleanup. +Hard timeouts are a last bound, not proof that a predecessor is already stopped. + +## 3. Implementation sequence and file map + +Implement in the isolated PR worktree, preserving unrelated work. Rebase/update only through the +approved PR workflow; do not reset active checkouts. Keep the already-landed review fixes during the +rewrite. The phases below are reviewable commits, not independently deployable releases. + +### Phase A — Establish identities and shared Modal construction + +- `packages/shared/src/types/integrations.ts`: add `modal-vm` to the backend enum and existing + resources/timeout capabilities. Remove feature-only Docker-setting capability plumbing when its + callers are removed. Do not equate that UI-setting capability with actual Docker support + elsewhere. +- `packages/control-plane/src/sandbox/provider-name.ts`, `provider-factory.ts`, and + `providers/modal-provider.ts`: configure one implementation with the exact immutable backend ID; + extend factory overloads and error/log identity consistently. Preserve HMAC transport and + credentials. +- `packages/control-plane/src/image-builds/model.ts` and `provider-factory.ts`: add `modal-vm` to + supported build identities and select the shared adapter using the recorded identity. +- Audit shared API schemas, node configuration, web provider parsing, fixtures, and all exhaustive + provider switches. Distinguish compute backend IDs from vendor-specific bundle targets: a sandbox + image bundle may still use target `modal` for both offerings because installation is shared. +- Extend the explicit Modal dashboard guards in `session/components.ts` and + `session/sandbox-access.ts` for both identities, without adding a provider-family registry. +- Add factory/identity tests before changing orchestration. Unknown providers must still fail; + missing configuration must still select ordinary `modal`. + +### Phase B — Own VM launch behavior inside Modal + +- `packages/control-plane/src/sandbox/client.ts` and `providers/modal-provider.ts`: implement the + private request/response contract and confirmation/cleanup behavior for all three allocation + paths. +- `packages/modal-infra/src/web_api.py`: validate the selector and removed legacy settings; derive + response identity from launch results. Preserve existing authentication and request validation. +- Refactor `packages/modal-infra/src/sandbox/docker_launch.py` into a focused Modal launch-policy + module (suggested name `launch_policy.py`), hiding image selection, VM runtime options, trusted + Docker signal, and resource defaults. Internal Docker service names may remain Docker-specific. +- `sandbox/manager.py`, `sandbox/build_session.py`: share policy/resource mapping and immutable + launch results; do not fork the lifecycle manager. Preserve explicit missing-image errors. +- Include the backend in deterministic allocation ownership/names or an equally strong local + ownership discriminator. Replace the internal artifact-variant tag with an explicit backend tag. + Do not adopt/terminate allocations solely by a guessed name or relabel old tags as newly verified. + With accepted orphan semantics, no legacy-adoption fallback is needed. +- Keep trusted Docker enablement derived from provider policy, overriding untrusted environment + values. Standard launches explicitly disable the Docker runtime service. +- Keep VM-specific deterministic adoption/retirement local. Do not expand all providers' generic + interfaces to describe Modal lookup races. + +**Mandatory preserved regressions from commit `5764ad45b`:** + +1. Retried/racing adoption returns the original allocation's interactive access credentials. Never + return newly generated credentials that were not installed; credential recovery failure is + closed. +2. Incompatible image-build allocation responses are durably bound for cleanup, rejected before + start, and cleaned up through the existing workflow/reaper. Malformed confirmation must not + discard a valid allocation ID. Failed session cleanup carries the allocation ID to + generation-pinned storage. +3. Orphan predecessor retirement is confirmed (`terminate(..., wait=True)` or equivalent proven + completion) before a successor can be created. Pending/failed retirement cannot launch a + successor. + +### Phase C — Remove the cross-layer Docker/variant machinery + +- Delete `packages/control-plane/src/sandbox/modal-docker.ts`; replace its necessary behavior with + Modal-local tests, not another shared helper importing Modal policy into orchestration. +- Remove its imports and freeze/admission/variant branches in: + - `src/scheduler/scheduler.ts`; + - `src/routes/session-create.ts`, `session-child-spawn.ts`; + - `src/session/initialize.ts`, integration-settings resolution, child spawn context/handler; + - `src/sandbox/lifecycle/manager.ts`, `src/sandbox/settings.ts`; + - settings persistence/routes and image-build policy/planning/workflow modules. +- Remove `dockerEnabled` from shared sandbox settings, session-create/override types, and public API + forwarding. New API writes containing it receive a targeted validation error; scope that check to + actual settings containers rather than recursively banning the string in arbitrary user data. +- Remove special child CPU/memory inheritance introduced only to freeze Docker mode. Retain the + pre-existing generic child/session settings policy and add regressions demonstrating it. +- Remove `snapshot_artifact_variant` read/write arguments from session ports, repository, shutdown, + and lifecycle policy. Preserve all generation, receipt-provider, checkpoint, and retirement + checks. +- Pass existing predecessor IDs as neutral create/restore context regardless of backend. The Modal + VM provider owns deterministic retirement; generic lifecycle never interprets a Docker flag. +- Do not spread `if (provider === "modal-vm")` into these modules. Provider composition, config + validation, and provider-specific adapters are the places allowed to discriminate. + +### Phase D — Simplify the complete image-build flow + +- `src/image-builds/{scope,planner,types,workflow,scheduler,lookup,provider-policy}.ts`: remove + variant selection, Docker admission, variant race checks, and `modal-docker-v1` literals. Keep + registration, secrets invalidation, callback authentication, runtime floors, timeouts, and source + cleanup. +- `src/image-builds/modal-adapter.ts` and Modal trigger config: pass only neutral resource requests; + backend selection comes from the constructed provider. Keep create-bind-confirm-start ordering; + bind records cleanup responsibility, not readiness. +- `src/db/image-builds.ts`: remove active variant arguments/columns from queries and row models, + after the migration strategy below is satisfied. Retain backend predicates on lookup, in-flight + deduplication, completion, and supersession. +- Prove independent `modal` and `modal-vm` rows can exist for the same scope without either being + selected as or superseding the other. A scope-wide secrets/config invalidation may intentionally + invalidate both; do not replace that safety rule with provider-only invalidation. +- Verify callback/finalization/cleanup route by the build's stored provider, not the deployment + default. Switching the default must not reinterpret an in-flight build as the new backend. + +### Phase E — Remove toggle UI and simplify deployment configuration + +- Remove `packages/web/src/components/docker-mode-select.tsx` and its feature-only wiring in + settings, composer, create-session API forwarding, and warm drafts. Retain generic CPU/memory + inputs for both Modal offerings. Show `Modal VM` where the application displays a backend label; + do not add a session/backend picker in this change. +- `terraform/environments/production/variables.tf`: accept `modal-vm` and apply Modal credential + validations to both identities. Remove the public `provision_modal_vm_sandboxes` and + `enable_modal_vm_sandboxes` inputs introduced by this PR. +- `locals.tf`: treat both IDs as Modal-family provisioning. Derive VM-image provisioning from + `sandbox_provider == "modal-vm"`. Keep the same module address/count for switching between them. +- `modal.tf`, `terraform/modules/modal-app/`, and deployment script: pass the derived need to build + the verified VM image. A private build flag is fine; it is not an independent operator selection. + Keep one app and the common base image; VM builds extend that image with Docker. +- `workers-control-plane.tf`, `src/types.ts`, `src/node/config.ts`: remove + `ENABLE_MODAL_VM_SANDBOXES`; propagate the exact `SANDBOX_PROVIDER` consistently to server and UI. + Keep the existing worker dependency on the Modal deployment module. +- `.github/workflows/terraform.yml`: remove both old VM flag mappings from plan and apply jobs; + backend identity alone drives provisioning. Update workflow-contract and Terraform tests. +- `.env.example`, deployment examples, and getting-started/provider docs: document the two + offerings, one selector, resource defaults, failure behavior, cleanup obligations, and unsupported + continuity. Node deployments select the same backend but must provision compatible Modal endpoints + themselves. +- `packages/modal-infra/deploy.py` and `src/images/base.py`: reuse the provider-local resource + defaults in VM verification. Fail deployment if a required VM image was not built/verified. Do not + let an old cache record stand in for the requested current image. Do not automatically delete old + images. + +## 4. Persistence and compatibility strategy + +**First determine whether the existing feature has actually been applied anywhere.** An open PR or +local migration file is not proof that it has or has not been deployed. Record the answer per target +environment before choosing a migration strategy. No live data changes are part of writing this +plan. + +### Preferred case: feature migrations/settings never deployed + +- Remove PR-only D1 migration `0081_image_build_artifact_variant.sql`, provided it is unpublished + and unapplied in every supported environment. Do not renumber unrelated migrations. +- Remove the PR-only session schema addition/migration for `snapshot_artifact_variant` where safe. +- Remove feature-only schemas/tests/fixtures; retain old ordinary Modal image rows under `modal`. +- No data backfill or session-provider pinning is needed. Existing provider columns are text; check + application enums and any actual database constraints rather than assuming the enum update + suffices. + +### If an environment used the variant feature + +- Preserve applied migration history. Leave obsolete columns unused initially; dropping them is not + needed to remove the abstraction from the active code. Add a forward-only migration/runbook for + that deployment instead of modifying an applied migration or rebuilding all tables unnecessarily. +- Before releasing queries without variant filtering, stop old writers and drain or fail legacy + in-flight variant builds. Mark legacy VM/unknown-variant artifacts unselectable using existing + failed/superseded lifecycle states, preserving image IDs, allocation IDs, and cleanup metadata. + Block late callbacks from restoring their readiness. Ordinary `default` Modal artifacts may + remain. +- Do **not** convert `provider=modal, artifact_variant=modal-docker-v1` to `provider=modal-vm` and + call it verified. Rebuild under the new confirmed backend identity. Accepted session orphaning + does not make mislabeled prepared images safe. +- Remove only the obsolete `dockerEnabled` property from known global/repository/environment + settings locations through a scoped, tested migration or operator edit. Preserve explicit CPU, + memory, timeout, and unrelated settings. Existing session records need not be migrated for + continuity; do not keep interpreting their old flag as a backend override. +- Inventory stale browser drafts and API clients. Drop the obsolete property from restored local + drafts; reject stale API writes with guidance to configure `SANDBOX_PROVIDER`, not an opaque + error. +- Back up metadata before an authorized transition. Test the actual upgrade fixture, including + cleanup and delayed callbacks. If old writers cannot be stopped, do not remove filtering yet; + split a bounded compatibility transition instead of silently accepting incompatible images. + +## 5. VM runtime safety and remaining review work + +The identity split removes orchestration complexity; it does not itself prove Docker lifecycle +correctness. Reconcile every outstanding PR finding against the final head. The three original fixes +are preserved regressions; build recovery, probe isolation, and log hygiene are confirmed shipping +requirements in this change. Live Docker pause/resume is explicitly deferred. + +VM session snapshots use the existing **destructive snapshot** contract: the supervisor quiesces +Docker over a local control socket, the provider captures the filesystem, then waits for VM +retirement. Only then does the response confirm `sourceStopped`; subsequent work restores into a new +generation. Standard Modal snapshots are unchanged. This avoids a new general snapshot protocol. + +Validate these areas: + +- **Checkpoint preparation:** current ordinary provider capture calls filesystem snapshot directly. + Audit all ordinary and shutdown capture entry points against runtime Docker preparation. Require + bounded container/daemon quiescence before a Docker filesystem is captured, followed by confirmed + retirement for VM session captures. Build finalization already quiesces Docker and retires its + source through the build workflow. Unknown outcomes stay fenced. Do not add live pause/resume + machinery. +- **Build allocation recovery:** test loss of the create response before provider-session binding. + Existing build ownership tags alone are not proof that create is idempotent. Make a deterministic + owned build allocation discoverable to the existing unbound-source recovery/cleanup contract, with + exact build identity checks; do not introduce a generic allocation framework. +- **Docker health checks:** ensure provider health checks target the local daemon despite user + `DOCKER_HOST`, context, TLS, or related environment settings. Sanitize the probe environment + without logging secrets or globally rewriting the user's workload environment. +- **Snapshot hygiene:** prevent daemon logs, stale credentials, and build-only auth material from + being baked into reusable images. Test cleanup failures and verify no secret-bearing diagnostic + output is exposed. Preserve host-runtime ownership and foreground daemon supervision. + +These are verification/closure requirements for shipping the VM offering, not claims that every +listed defect remains present or instructions to expand the identity refactor without review. If an +item requires a significant new lifecycle protocol, isolate it as a prerequisite and report the +scope change. Do not mark the overall VM feature ready based solely on provider factory tests. + +Snapshots preserve supported filesystem state, not process/RAM continuity or automatically +application-consistent databases. Document precisely which Docker state the canary actually proves. + +## 6. Validation plan + +### Automated coverage + +| Boundary | Required evidence | +| ---------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Identity/configuration | Missing selection is `modal`; `modal-vm` parses everywhere; invalid values fail; both use shared Modal credentials and the correct name/capabilities. | +| Sessions/children/automation | Scheduler and create/child paths contain no Docker policy; existing generic settings still work; no public runtime override can alter the deployment choice. | +| Modal create/restore/build | Exact backend selector and launch kwargs; correct base/prebuilt image; trusted Docker signal; VM missing-image failure; explicit/custom/null/default resource cases. | +| Rolling deployment | Old endpoint ignoring new fields fails closed for every VM allocation path; known allocations cleaned up; cleanup failures observable; legacy standard responses retain compatibility. | +| Allocation races | Owned adoption preserves credentials; wrong-owner/backend allocations cannot be adopted or retired; predecessor wait gates successor; lost build-create response can be reconciled. | +| Image build lifecycle | VM confirmation precedes start; handles are bound first for durable cleanup; both backend rows coexist; lookup/supersession/dedupe partition correctly; callbacks and cleanup still use recorded backend after config change. | +| Persistence | Fresh schema works without variant fields; applied-feature upgrade fixture cannot expose legacy VM artifacts as standard; late callbacks cannot reactivate quarantined builds. | +| Snapshot/lifecycle | Existing receipt-provider and generation checks survive; quiescence and unknown-outcome behavior are tested at the assembled lifecycle boundary, not only helper level. | +| Runtime/UI | Local Docker probe isolation, Docker startup/stop/hygiene; toggle absent; restored drafts cleaned; generic resource inputs and ordinary Modal behavior retained. | +| Terraform/workflows | Both IDs provision the same Modal app; only VM selection requires VM image verification; plan/apply env mapping agrees; switch does not accidentally replace the app; other providers unchanged. | + +Run targeted regressions first, then the relevant full suites. Build shared before dependent checks: + +```bash +npm run build -w @open-inspect/shared +npm run typecheck -w @open-inspect/control-plane -w @open-inspect/web +npm test -w @open-inspect/control-plane +npm run test:integration -w @open-inspect/control-plane +npm test -w @open-inspect/web +npm run lint -w @open-inspect/control-plane +``` + +From each Python package (`packages/modal-infra`, `packages/sandbox-runtime`, and +`packages/sandbox-images`), run its supported test/lint commands in the project environment, +including `pytest tests/`, `ruff check`, and `ruff format --check`. Run the repository's Terraform +mock/contract tests, validate the affected Terraform, and check formatting of changed files. Confirm +installed tools and package scripts before constructing exact commands; avoid live deploy hooks in +validation. Record exact-head pass/fail evidence and distinguish pre-existing failures from new +ones. + +### Structural acceptance checks + +- Search executable shared/orchestration code for `modal-docker`, `artifactVariant`, + `snapshot_artifact_variant`, `modal-docker-v1`, and Docker admission helpers: no active policy + remains. Applied migration history, explicit legacy-input rejection, and upgrade fixtures are + documented exceptions, not reasons to keep the concept in new contracts. +- Search `dockerEnabled`: no new persisted session setting or runtime selector remains. Provider + response compatibility and targeted legacy validation must be named/documented exceptions. +- Search `modal-vm`: occurrences in generic scheduler/session lifecycle logic need justification; + expected homes are provider registries, composition/configuration, adapters, tests, and + documentation. +- No duplicated Modal provider/lifecycle implementation, mutable client mode, speculative routing + framework, or new session backend pinning. +- Every removed safety check has a replacement invariant/test or an explicit accepted non-goal; do + not erase runtime/version/ownership safeguards along with variant plumbing. + +## 7. Rollout, canary, and rollback + +Implementation and deployment are separate approvals. Do not run billable Modal allocations, +Terraform apply, remote data migrations, or production cleanup simply to finish the code change. + +For an authorized rollout: + +1. Determine the persistence case in section 4. Record known active sessions/builds, cleanup + obligations, old/new image IDs, endpoint deployment revision, and selected backend without + secrets. +2. Deploy the compatible Modal API and verified required image first. Keep the existing worker + dependency ordering; independently deployed node workers must follow the same ordering. +3. For a cutover from an already deployed variant feature, stop old writers and perform the approved + scoped metadata/settings transition before removing variant filters. Do not run old/new writers + concurrently against ambiguous legacy artifact rows. +4. Deploy control plane/UI with the explicit backend selection. `modal` is the default; selecting + `modal-vm` is the operator's opt-in. A missing/old VM endpoint must surface a clear failure. +5. On an authorized staging deployment, exercise a fresh VM session, a prepared-image build and + session launch from it, IDE/VNC credentials after adoption, ordinary checkpoint/restore, and + confirmed shutdown/replacement. Prove a user Docker container can run and persist the specific + filesystem state promised. Inspect cleanup after intentional launch/confirmation failures. +6. Independently exercise standard `modal` with no VM option/Docker service, plus image build and + restore. Record concrete allocation/runtime evidence; mocks alone do not prove provider behavior. +7. Observe launch failures, unexpected runtime confirmations, cleanup backlog, duplicate + allocations, build completion, and leaked resources under each distinct backend label. + +Rollback selects `modal` and deploys a compatible control plane/data plane, or restores the prior +known-good application version subject to the migration constraints. It is **not** transparent VM +session failover. Do not repurpose VM artifacts as standard images, revert applied migrations, or +remove cleanup credentials/endpoints while recorded resources still need reclamation. If restoring +the old variant implementation after a data transition, reconcile its writers/settings/artifact +expectations explicitly; a code rollback alone is insufficient. + +## 8. Follow-up: mixed selection without restoring variants + +When mixed workloads become an actual requirement, retain these same concrete backend IDs. Add a +neutral selection policy (for example automation/repository default plus an authorized session +override), resolve the backend once, and carry/persist that choice through session construction, +children, image selection, cleanup, and recovery. At that point define pinning and +configuration-change semantics deliberately. Provisioning both offerings can then be independent +from choosing the default. + +That follow-up should route **to an existing provider instance/identity**, not recreate a +`dockerEnabled` flag that scheduler, images, and lifecycle must interpret. Cheap review workloads +and VM/Docker workloads can coexist without a new artifact-compatibility dimension. No selector, +pinning column, routing policy, or multi-offering Terraform interface is implemented speculatively +now. + +## 9. Completion criteria and handoff + +- Both backend identities work through one Modal implementation and the existing generic contracts. +- Active variant plumbing and session Docker-toggle scaffolding are removed, not renamed. +- Generic resources remain generic; Modal VM defaults and launch mechanics have a provider-local + owner. +- Backend-specific artifacts cannot cross through lookup, late callbacks, rollout, or fallback. +- All three fixed review regressions remain covered, outstanding VM safety findings are reconciled, + and relevant automated checks pass or have clearly documented baseline failures. +- Migration/deployment assumptions are explicit and verified before any rollout; accepted session + orphaning is documented without claiming automatic retirement or continuity. +- Mixed selection is documented only as a follow-up. No production deployment or destructive cleanup + is implied by code completion. +- Update the existing PR with the decision, removed concepts, test evidence, and rollout + requirements when implementation is authorized. Do not resolve review threads merely because the + plan mentions them; resolve against implemented and verified code. diff --git a/packages/control-plane/src/image-builds/modal-adapter.ts b/packages/control-plane/src/image-builds/modal-adapter.ts index ecc9050888..ade5ded19d 100644 --- a/packages/control-plane/src/image-builds/modal-adapter.ts +++ b/packages/control-plane/src/image-builds/modal-adapter.ts @@ -17,12 +17,20 @@ import { resolveImageBuildProviderSessionTimeoutSeconds } from "./timeouts"; * Modal provider-session image build adapter. */ export class ModalImageBuildAdapter implements ImageBuildAdapter { - constructor(private readonly provider: ModalImageBuildProvider) {} + readonly recoverUnboundSource?: ImageBuildAdapter["recoverUnboundSource"]; + + constructor(private readonly provider: ModalImageBuildProvider) { + if (provider.recoverImageBuildSandbox) { + this.recoverUnboundSource = (input) => + provider.recoverImageBuildSandbox!(input.buildId, input.correlation, input.signal); + } + } async startBuild(plan: ImageBuildPlan, callbacks: ImageBuildStartCallbacks): Promise { await this.provider.triggerImageBuild({ scopeKind: plan.scope.kind, scopeId: plan.scope.id, + resources: plan.resources, buildId: plan.buildId, repositories: plan.repositories, cloneToken: plan.cloneAuth.type === "credential_helper" ? plan.cloneAuth.token : undefined, diff --git a/packages/control-plane/src/image-builds/model.ts b/packages/control-plane/src/image-builds/model.ts index 156a23ad83..febf497480 100644 --- a/packages/control-plane/src/image-builds/model.ts +++ b/packages/control-plane/src/image-builds/model.ts @@ -34,6 +34,7 @@ import { */ export const IMAGE_BUILD_PROVIDER_IDS = [ "modal", + "modal-vm", "vercel", "opencomputer", "e2b", diff --git a/packages/control-plane/src/image-builds/planner.ts b/packages/control-plane/src/image-builds/planner.ts index 7fab22e9fc..9a61f7fa8d 100644 --- a/packages/control-plane/src/image-builds/planner.ts +++ b/packages/control-plane/src/image-builds/planner.ts @@ -99,6 +99,7 @@ export class ImageBuildPlanner implements ImageBuildPlannerPort { callbackUrl: params.callbackUrl, failureCallbackUrl: params.failureCallbackUrl, buildTimeoutMs: resolveBuildTimeoutSeconds(sandboxSettings) * MS_PER_SECOND, + resources: { cpuCores: sandboxSettings.cpuCores, memoryMib: sandboxSettings.memoryMib }, userEnvVars: userEnvVars ? prepareLegacyManagedProviderEnv({ exposedSecrets: userEnvVars, diff --git a/packages/control-plane/src/image-builds/provider-factory.ts b/packages/control-plane/src/image-builds/provider-factory.ts index e91efa682d..0c9535e110 100644 --- a/packages/control-plane/src/image-builds/provider-factory.ts +++ b/packages/control-plane/src/image-builds/provider-factory.ts @@ -37,7 +37,8 @@ class EnvImageBuildAdapterFactory implements ImageBuildAdapterFactory { create(provider: ImageBuildProvider, operation: "start" | "existing_session"): ImageBuildAdapter { switch (provider) { case "modal": - return new ModalImageBuildAdapter(createSandboxProviderFromEnv(this.env, "modal")); + case "modal-vm": + return new ModalImageBuildAdapter(createSandboxProviderFromEnv(this.env, provider)); case "vercel": return new VercelImageBuildAdapter(createSandboxProviderFromEnv(this.env, "vercel")); case "opencomputer": diff --git a/packages/control-plane/src/image-builds/types.ts b/packages/control-plane/src/image-builds/types.ts index 9ba09a11ce..cc06769080 100644 --- a/packages/control-plane/src/image-builds/types.ts +++ b/packages/control-plane/src/image-builds/types.ts @@ -1,4 +1,5 @@ import type { RepositoryShaEntry } from "@open-inspect/shared/types/image-builds"; +import type { SandboxSettings } from "@open-inspect/shared/types/integrations"; import type { CorrelationContext } from "../logger"; import type { ImageBuildProviderImageRef, ImageBuildScope } from "./model"; @@ -32,6 +33,7 @@ export type ImageBuildCloneAuth = * Every supported provider uses the same create-bind-launch session contract. */ export interface ImageBuildPlan { + resources?: Pick; buildId: string; scope: ImageBuildScope; repositories: ImageBuildRepository[]; diff --git a/packages/control-plane/src/routes/image-builds.trigger.test.ts b/packages/control-plane/src/routes/image-builds.trigger.test.ts index 0d8f68bac1..878bb68fc2 100644 --- a/packages/control-plane/src/routes/image-builds.trigger.test.ts +++ b/packages/control-plane/src/routes/image-builds.trigger.test.ts @@ -205,6 +205,7 @@ const registerBuildSpy = vi.spyOn(ImageBuildStore.prototype, "registerBuild"); const getActiveBuildSpy = vi.spyOn(ImageBuildStore.prototype, "getActiveBuild"); const hasReadyImageSpy = vi.spyOn(ImageBuildStore.prototype, "hasReadyImageForFingerprint"); const markBuildFailedSpy = vi.spyOn(ImageBuildStore.prototype, "markBuildFailed"); +const markSourceCreateIntentSpy = vi.spyOn(ImageBuildStore.prototype, "markSourceCreateIntent"); const bindProviderSessionSpy = vi.spyOn(ImageBuildStore.prototype, "bindProviderSession"); const setImageBuildEnabledSpy = vi.spyOn(RepoMetadataStore.prototype, "setImageBuildEnabled"); @@ -220,6 +221,7 @@ beforeEach(() => { markBuildFailedSpy.mockResolvedValue(true); setImageBuildEnabledSpy.mockResolvedValue(undefined); bindProviderSessionSpy.mockResolvedValue(true); + markSourceCreateIntentSpy.mockResolvedValue(true); modalClient.createImageBuildSandbox.mockResolvedValue({ providerSessionId: "modal-session-1", }); diff --git a/packages/control-plane/src/sandbox/client.ts b/packages/control-plane/src/sandbox/client.ts index 70c4cd54e0..be658c23e6 100644 --- a/packages/control-plane/src/sandbox/client.ts +++ b/packages/control-plane/src/sandbox/client.ts @@ -16,6 +16,8 @@ import { buildSessionConfig, toRepositoryConfigPayload } from "./sandbox-env"; import type { SessionRepositoryInfo } from "./provider"; import { withRequestDeadline } from "./request-deadline"; +export type ModalBackend = "modal" | "modal-vm"; + const log = createLogger("modal-client"); // Modal app name @@ -36,6 +38,8 @@ const createSandboxModalResponseSchema = z.object({ data: z.object({ sandbox_id: z.string(), modal_object_id: z.string().nullable().optional(), + sandbox_backend: z.unknown().optional(), + docker_enabled: z.unknown().optional(), created_at: z.number(), code_server_url: z.string().nullable().optional(), code_server_password: z.string().nullable().optional(), @@ -51,6 +55,8 @@ const restoreSandboxModalResponseSchema = z.object({ data: z.object({ sandbox_id: z.string().min(1), modal_object_id: z.string().nullable().optional(), + sandbox_backend: z.unknown().optional(), + docker_enabled: z.unknown().optional(), code_server_url: z.string().nullable().optional(), code_server_password: z.string().nullable().optional(), vnc_url: z.string().nullable().optional(), @@ -64,6 +70,7 @@ const snapshotSandboxModalResponseSchema = z.object({ success: z.literal(true), data: z.object({ image_id: z.string().min(1), + source_stopped: z.boolean().optional(), }), }); @@ -72,6 +79,8 @@ const createImageBuildSandboxModalResponseSchema = z.object({ data: z.object({ // Non-empty: the previous hand-rolled check rejected a blank id. provider_session_id: z.string().min(1), + sandbox_backend: z.unknown().optional(), + docker_enabled: z.unknown().optional(), }), }); @@ -141,6 +150,8 @@ export function buildModalSandboxDashboardUrl(params: { } export interface CreateSandboxRequest { + sandboxBackend?: ModalBackend; + retireSandboxId?: string | null; sessionId: string; sandboxId?: string; // Expected sandbox ID (generated by control plane) repoOwner: string | null; @@ -166,6 +177,9 @@ export interface CreateSandboxRequest { } export interface CreateSandboxResponse { + /** Validated by the provider after retaining the allocation handle. */ + sandboxBackend?: unknown; + legacyDockerEnabled?: unknown; sandboxId: string; modalObjectId?: string; // Modal's internal object ID for snapshot API createdAt: number; @@ -178,6 +192,8 @@ export interface CreateSandboxResponse { } export interface RestoreSandboxRequest { + sandboxBackend?: ModalBackend; + retireSandboxId?: string | null; snapshotImageId: string; sessionId: string; sandboxId: string; @@ -201,6 +217,9 @@ export interface RestoreSandboxRequest { } export interface RestoreSandboxResponse { + /** Validated by the provider after retaining the allocation handle. */ + sandboxBackend?: unknown; + legacyDockerEnabled?: unknown; sandboxId: string; modalObjectId?: string; codeServerUrl?: string; @@ -225,6 +244,7 @@ export interface StopSandboxRequest { } export interface SnapshotSandboxResponse { + sourceStopped?: boolean; imageId: string; } @@ -235,6 +255,8 @@ export interface SnapshotBuildSandboxRequest { } export interface CreateImageBuildSandboxRequest { + resources?: Pick; + sandboxBackend?: ModalBackend; /** Scope kind ("repo" | "environment") — accepted by Modal for logging only. */ scopeKind: ImageBuildScopeKind; /** Scope id (lowercase owner/name or environment id) — logging only. */ @@ -255,6 +277,9 @@ export interface CreateImageBuildSandboxRequest { } export interface CreateImageBuildSandboxResponse { + /** Validated by the provider after retaining the allocation handle. */ + sandboxBackend?: unknown; + legacyDockerEnabled?: unknown; providerSessionId: string; } @@ -298,6 +323,7 @@ export class ModalClient { private restoreSandboxUrl: string; private stopSandboxUrl: string; private createImageBuildSandboxUrl: string; + private recoverImageBuildSandboxUrl: string; private startImageBuildSandboxUrl: string; private terminateImageBuildSandboxUrl: string; private secret: string; @@ -345,6 +371,7 @@ export class ModalClient { this.restoreSandboxUrl = url("api-restore-sandbox"); this.stopSandboxUrl = url("api-stop-sandbox"); this.createImageBuildSandboxUrl = url("api-create-build-sandbox"); + this.recoverImageBuildSandboxUrl = url("api-recover-build-sandbox"); this.startImageBuildSandboxUrl = url("api-start-build-sandbox"); this.terminateImageBuildSandboxUrl = url("api-terminate-build-sandbox"); } @@ -403,6 +430,8 @@ export class ModalClient { agent_slack_notify_enabled: request.agentSlackNotifyEnabled ?? false, mcp_servers: request.mcpServers || null, sandbox_settings: request.sandboxSettings ?? null, + sandbox_backend: request.sandboxBackend, + retire_sandbox_id: request.retireSandboxId, // Flat keys matching SessionConfig field names — Modal's create // handler builds its SessionConfig from the request by field name // (unlike restore, which carries a nested session_config). @@ -421,6 +450,8 @@ export class ModalClient { return { sandboxId: result.data.sandbox_id, modalObjectId: result.data.modal_object_id ?? undefined, + sandboxBackend: result.data.sandbox_backend, + legacyDockerEnabled: result.data.docker_enabled, createdAt: result.data.created_at, codeServerUrl: result.data.code_server_url ?? undefined, codeServerPassword: result.data.code_server_password ?? undefined, @@ -473,6 +504,8 @@ export class ModalClient { vnc_enabled: request.vncEnabled ?? false, agent_slack_notify_enabled: request.agentSlackNotifyEnabled ?? false, sandbox_settings: request.sandboxSettings ?? null, + sandbox_backend: request.sandboxBackend, + retire_sandbox_id: request.retireSandboxId, }, restoreSandboxModalResponseSchema, correlation, @@ -484,6 +517,8 @@ export class ModalClient { return { sandboxId: result.data.sandbox_id, modalObjectId: result.data.modal_object_id ?? undefined, + sandboxBackend: result.data.sandbox_backend, + legacyDockerEnabled: result.data.docker_enabled, codeServerUrl: result.data.code_server_url ?? undefined, codeServerPassword: result.data.code_server_password ?? undefined, vncUrl: result.data.vnc_url ?? undefined, @@ -538,7 +573,7 @@ export class ModalClient { (status) => (httpStatus = status) ); outcome = "success"; - return { imageId: result.data.image_id }; + return { imageId: result.data.image_id, sourceStopped: result.data.source_stopped }; } finally { log.info("modal.request", { event: "modal.request", @@ -594,7 +629,7 @@ export class ModalClient { (status) => (httpStatus = status) ); outcome = "success"; - return { imageId: result.data.image_id }; + return { imageId: result.data.image_id, sourceStopped: result.data.source_stopped }; } finally { log.info("modal.request", { event: "modal.request", @@ -625,6 +660,8 @@ export class ModalClient { endpoint, MODAL_SANDBOX_START_REQUEST_DEADLINE_MS, { + sandbox_backend: request.sandboxBackend, + sandbox_settings: request.resources, scope_kind: request.scopeKind, scope_id: request.scopeId, build_id: request.buildId, @@ -647,6 +684,8 @@ export class ModalClient { outcome = "success"; return { providerSessionId: result.data.provider_session_id, + sandboxBackend: result.data.sandbox_backend, + legacyDockerEnabled: result.data.docker_enabled, }; } finally { log.info("modal.request", { @@ -664,6 +703,30 @@ export class ModalClient { } } + async recoverImageBuildSandbox( + buildId: string, + sandboxBackend: ModalBackend, + correlation?: CorrelationContext, + signal?: AbortSignal + ): Promise<{ providerSessionId: string } | null> { + const response = await this.postJson( + this.recoverImageBuildSandboxUrl, + "recoverImageBuildSandbox", + MODAL_CLEANUP_REQUEST_DEADLINE_MS, + { build_id: buildId, sandbox_backend: sandboxBackend }, + z.object({ + success: z.literal(true), + data: z.object({ provider_session_id: z.string().min(1).nullable() }), + }), + correlation, + signal, + () => {} + ); + return response.data.provider_session_id + ? { providerSessionId: response.data.provider_session_id } + : null; + } + async startImageBuildSandbox( request: StartImageBuildSandboxRequest, correlation?: CorrelationContext diff --git a/packages/control-plane/src/sandbox/lifecycle/manager.test.ts b/packages/control-plane/src/sandbox/lifecycle/manager.test.ts index 3e7daf2fe3..b968bbb176 100644 --- a/packages/control-plane/src/sandbox/lifecycle/manager.test.ts +++ b/packages/control-plane/src/sandbox/lifecycle/manager.test.ts @@ -22,6 +22,7 @@ import { PrebuiltImageActivationPendingError, PrebuiltImageUnavailableError, SandboxProviderError, + SandboxLaunchRejectedError, type SandboxProvider, type CreateSandboxConfig, type CreateSandboxResult, @@ -4728,27 +4729,33 @@ describe("status writes after a provider await (COL-99)", () => { } }); - it("counts an attempt once when the watchdog fails it before the provider rejects it", async () => { - // The connect alarm is armed at reservation, before the provider call, - // so it can fail the attempt while createSandbox() is still pending. The - // provider's later rejection is the same attempt, not a second failure. - vi.useFakeTimers(); - try { - const sandbox = createMockSandbox({ status: "failed" }); - const h = harness(sandbox, async () => { - vi.advanceTimersByTime(DEFAULT_LIFECYCLE_CONFIG.connectingTimeout.timeoutMs + 1000); - await expect(h.manager.handleAlarm()).resolves.toBe("sandbox_failed"); - throw new SandboxProviderError("quota exceeded", "permanent"); - }); + it.each([ + new SandboxProviderError("quota exceeded", "permanent"), + new SandboxLaunchRejectedError("incompatible", "sb-rejected"), + ])( + "counts an attempt once when the watchdog fails it before the provider rejects it (%s)", + async (error) => { + // The connect alarm is armed at reservation, before the provider call, + // so it can fail the attempt while createSandbox() is still pending. The + // provider's later rejection is the same attempt, not a second failure. + vi.useFakeTimers(); + try { + const sandbox = createMockSandbox({ status: "failed" }); + const h = harness(sandbox, async () => { + vi.advanceTimersByTime(DEFAULT_LIFECYCLE_CONFIG.connectingTimeout.timeoutMs + 1000); + await expect(h.manager.handleAlarm()).resolves.toBe("sandbox_failed"); + throw error; + }); - await h.manager.spawnSandbox(); + await h.manager.spawnSandbox(); - expect(sandbox.status).toBe("failed"); - expect(sandbox.spawn_failure_count).toBe(1); - } finally { - vi.useRealTimers(); + expect(sandbox.status).toBe("failed"); + expect(sandbox.spawn_failure_count).toBe(1); + } finally { + vi.useRealTimers(); + } } - }); + ); it("leaves a sandbox whose bridge attached during the provider call booting when the call then fails", async () => { // The bridge now connects ahead of its boot, so a provider error that diff --git a/packages/control-plane/src/sandbox/lifecycle/manager.ts b/packages/control-plane/src/sandbox/lifecycle/manager.ts index 518321537a..ac5731375e 100644 --- a/packages/control-plane/src/sandbox/lifecycle/manager.ts +++ b/packages/control-plane/src/sandbox/lifecycle/manager.ts @@ -34,6 +34,7 @@ import { import { PrebuiltImageUnavailableError, SandboxProviderError, + SandboxLaunchRejectedError, type SandboxProvider, type CreateSandboxConfig, type CreateSandboxResult, @@ -211,6 +212,7 @@ export interface SandboxStorage { * store its handle, and advance a fresh spawn to connecting. Returns the * resulting status, or null when another lifecycle event owns the row. */ + rejectProviderStartup(generation: SandboxGeneration, providerObjectId: string | null): boolean; commitProviderStartup( generation: SandboxGeneration, providerObjectId: string | null, @@ -703,6 +705,10 @@ export class SandboxLifecycleManager const now = Date.now(); const sessionId = session.session_name || session.id; const hasRepository = sessionHasRepository(session); + const priorSandbox = this.storage.getSandbox(); + const priorSandboxId = priorSandbox?.modal_sandbox_id ?? null; + // A fenced allocation must be retired before its durable identity is replaced. + if (priorSandbox?.fenced) await this.stopPriorProviderSandbox(true); const reserved = this.spawnGeneration(session, now); generation = reserved; let { sandboxAuthToken, expectedSandboxId } = await this.reserveSpawnIdentity(reserved, { @@ -753,6 +759,7 @@ export class SandboxLifecycleManager const timeoutSeconds = this.resolveSandboxTimeoutSeconds(sandboxSettings); const createConfig: CreateSandboxConfig = { sessionId, + retireSandboxId: priorSandboxId, sandboxId: expectedSandboxId, repoOwner: session.repo_owner, repoName: session.repo_name, @@ -852,6 +859,7 @@ export class SandboxLifecycleManager }); return; } + await this.retainRejectedAllocation(error, generation); const errorMessage = error instanceof Error ? error.message : "Failed to spawn sandbox"; this.log.error("Sandbox spawn completed", { event: "sandbox.spawn", @@ -1129,6 +1137,10 @@ export class SandboxLifecycleManager this.storage.setLastSpawnError(null, null); const now = Date.now(); + const priorSandbox = this.storage.getSandbox(); + const priorSandboxId = priorSandbox?.modal_sandbox_id ?? null; + // A fenced allocation must be retired before its durable identity is replaced. + if (priorSandbox?.fenced) await this.stopPriorProviderSandbox(true); const reserved = this.spawnGeneration(session, now); generation = reserved; const shutdownPolicy = shutdownPolicyForLaunch("existing", snapshotRuntimeVersion); @@ -1158,6 +1170,7 @@ export class SandboxLifecycleManager if (restoringSavedState) this.shutdown.markRecoveryInvoked(generation); const result = await this.provider.restoreFromSnapshot({ snapshotImageId, + retireSandboxId: priorSandboxId, sessionId: session.session_name || session.id, sandboxId: expectedSandboxId, sandboxAuthToken, @@ -1240,6 +1253,7 @@ export class SandboxLifecycleManager }); return; } + await this.retainRejectedAllocation(error, generation); const errorMessage = error instanceof Error ? error.message : "Failed to restore sandbox"; this.log.error("Sandbox restore completed", { event: "sandbox.restore", @@ -1413,7 +1427,7 @@ export class SandboxLifecycleManager /** * Stop a sandbox that is about to be replaced before its provider handle is cleared. */ - private async stopPriorProviderSandbox(): Promise { + private async stopPriorProviderSandbox(requireConfirmation = false): Promise { const providerObjectId = this.storage.getSandbox()?.modal_object_id; if (!providerObjectId) { return; @@ -1439,6 +1453,7 @@ export class SandboxLifecycleManager ]); this.storage.updateSandboxModalObjectId(null); } catch (error) { + if (requireConfirmation) throw error; this.storage.updateSandboxModalObjectId(null); this.log.warn("Provider stop failed before sandbox replacement", { provider_object_id: providerObjectId, @@ -2147,6 +2162,25 @@ export class SandboxLifecycleManager await this.storage.updateSandboxAccess("ttyd", url, token); } + private async retainRejectedAllocation( + error: unknown, + generation: SandboxGeneration | null + ): Promise { + if (!(error instanceof SandboxLaunchRejectedError) || !generation) return; + const alreadyFailed = this.storage.getSandbox()?.status === "failed"; + // Keep the rejected handle on this generation so restart/replacement retries retirement. + // It is not a successful startup and receives no access credentials or readiness signal. + if (!this.storage.rejectProviderStartup(generation, error.providerObjectId)) { + await this.destroyLateProviderResult(error.providerObjectId ?? undefined); + return; + } + this.wsManager.detachSandboxWebSocket(1008, "Provider allocation rejected"); + this.clearSandboxAccessState(); + this.broadcaster.broadcast({ type: "sandbox_status", status: "failed" }); + this.reportSandboxError(error.message); + if (!alreadyFailed) this.recordSpawnFailure(Date.now(), generation.createdAt); + } + private async claimProviderStartup( generation: SandboxGeneration, providerObjectId: string | undefined, diff --git a/packages/control-plane/src/sandbox/lifecycle/rejected-allocation.test.ts b/packages/control-plane/src/sandbox/lifecycle/rejected-allocation.test.ts new file mode 100644 index 0000000000..f7a2c29b2f --- /dev/null +++ b/packages/control-plane/src/sandbox/lifecycle/rejected-allocation.test.ts @@ -0,0 +1,58 @@ +import { describe, expect, it, vi } from "vitest"; +import { SandboxLaunchRejectedError } from "../provider"; +import { createAlarmFixture, createMockProvider, createMockSandbox } from "./test-helpers"; + +describe("rejected provider allocation", () => { + it.each([ + [false, "sb-rejected"], + [true, "sb-rejected"], + ["expired", "sb-rejected"], + [false, null], + [true, null], + ["expired", null], + ] as const)( + "retains cleanup identity and fences the rejected generation (early bridge=%s, cleanup handle=%s)", + async (earlyBridge, cleanupHandle) => { + const sandbox = createMockSandbox({ status: "pending", modal_object_id: null }); + const provider = createMockProvider({ + capabilities: { supportsExplicitStop: true }, + createSandbox: async () => { + if (earlyBridge) { + sandbox.status = earlyBridge === "expired" ? "failed" : "connecting"; + if (earlyBridge === "expired") sandbox.fenced = 1; + vi.mocked(fixture.wsManager.getSandboxWebSocket).mockReturnValue({} as WebSocket); + } + throw new SandboxLaunchRejectedError("incompatible allocation", cleanupHandle); + }, + }); + const fixture = createAlarmFixture(sandbox, provider); + await fixture.manager.spawnSandbox(); + expect(sandbox.status).toBe("failed"); + expect(sandbox.fenced).toBe(1); + expect(sandbox.auth_token_hash).toBe(""); + expect(sandbox.modal_object_id).toBe(cleanupHandle); + expect(fixture.wsManager.detachSandboxWebSocket).toHaveBeenCalled(); + } + ); + it("retains rejected cleanup responsibility across restart and failed retirement", async () => { + const sandbox = createMockSandbox({ status: "pending", modal_object_id: null }); + const provider = createMockProvider({ + capabilities: { supportsExplicitStop: true }, + createSandbox: vi.fn(async () => { + throw new SandboxLaunchRejectedError("incompatible", "sb-rejected"); + }), + stopSandbox: vi.fn(async () => { + throw new Error("provider unavailable"); + }), + }); + await createAlarmFixture(sandbox, provider).manager.spawnSandbox(); + const rejectedGeneration = sandbox.modal_sandbox_id; + const restarted = createAlarmFixture(sandbox, provider); + await restarted.manager.spawnSandbox(); + await restarted.manager.spawnSandbox(); + expect(sandbox.modal_object_id).toBe("sb-rejected"); + expect(sandbox.modal_sandbox_id).toBe(rejectedGeneration); + expect(sandbox.fenced).toBe(1); + expect(provider.createSandbox).toHaveBeenCalledTimes(1); + }); +}); diff --git a/packages/control-plane/src/sandbox/lifecycle/test-helpers.ts b/packages/control-plane/src/sandbox/lifecycle/test-helpers.ts index 9f62d074ac..85e8a6f4f7 100644 --- a/packages/control-plane/src/sandbox/lifecycle/test-helpers.ts +++ b/packages/control-plane/src/sandbox/lifecycle/test-helpers.ts @@ -162,6 +162,21 @@ export function createMockStorage( return true; } ), + rejectProviderStartup: vi.fn((generation, providerObjectId) => { + if ( + !sandbox || + sandbox.modal_sandbox_id !== generation.sandboxId || + sandbox.created_at !== generation.createdAt + ) + return false; + sandbox.modal_object_id = providerObjectId; + sandbox.status = "failed"; + sandbox.fenced = 1; + sandbox.auth_token_hash = ""; + sandbox.auth_token = null; + sandbox.active_socket_id = ""; + return true; + }), commitProviderStartup: vi.fn((generation, providerObjectId, allowFailedSelfHeal) => { calls.push("commitProviderStartup"); if ( diff --git a/packages/control-plane/src/sandbox/provider-factory.ts b/packages/control-plane/src/sandbox/provider-factory.ts index 271aaa0dbc..54b3b02c15 100644 --- a/packages/control-plane/src/sandbox/provider-factory.ts +++ b/packages/control-plane/src/sandbox/provider-factory.ts @@ -21,7 +21,7 @@ import { createVercelProvider, type VercelSandboxProvider } from "./providers/ve import { resolveScmProviderFromEnv } from "../source-control"; import type { Env } from "../types"; -function createModalProviderFromEnv(env: Env): ModalSandboxProvider { +function createModalProviderFromEnv(env: Env, backend: "modal" | "modal-vm"): ModalSandboxProvider { if (!env.MODAL_API_SECRET || !env.MODAL_WORKSPACE) { throw new Error( "MODAL_API_SECRET and MODAL_WORKSPACE are required when SANDBOX_PROVIDER=modal" @@ -35,7 +35,7 @@ function createModalProviderFromEnv(env: Env): ModalSandboxProvider { env.MODAL_API_URL ); - return createModalProvider(client); + return createModalProvider(client, backend); } function createVercelProviderFromEnv(env: Env): VercelSandboxProvider { @@ -170,7 +170,10 @@ function createE2BProviderFromEnv(env: Env): E2BSandboxProvider { export function createSandboxProviderFromEnv(env: Env, backend: "daytona"): DaytonaSandboxProvider; export function createSandboxProviderFromEnv(env: Env, backend: "e2b"): E2BSandboxProvider; -export function createSandboxProviderFromEnv(env: Env, backend: "modal"): ModalSandboxProvider; +export function createSandboxProviderFromEnv( + env: Env, + backend: "modal" | "modal-vm" +): ModalSandboxProvider; export function createSandboxProviderFromEnv(env: Env, backend: "vercel"): VercelSandboxProvider; export function createSandboxProviderFromEnv( env: Env, @@ -199,7 +202,8 @@ export function createSandboxProviderFromEnv( case "e2b": return createE2BProviderFromEnv(env); case "modal": - return createModalProviderFromEnv(env); + case "modal-vm": + return createModalProviderFromEnv(env, backend); } } diff --git a/packages/control-plane/src/sandbox/provider.ts b/packages/control-plane/src/sandbox/provider.ts index bec8f1bd09..3baaccf0e0 100644 --- a/packages/control-plane/src/sandbox/provider.ts +++ b/packages/control-plane/src/sandbox/provider.ts @@ -145,6 +145,8 @@ export interface CreateSandboxConfig { mcpServers?: McpServerConfig[]; /** Sandbox settings (tunnel ports, etc.) resolved from integration settings */ sandboxSettings?: SandboxSettings; + /** Previous logical allocation identity, used by providers supporting ambiguous-create recovery. */ + retireSandboxId?: string | null; /** * Ordered member list for multi-repo sessions. Only set when the session * has more than one member — single-repo sessions keep the scalar @@ -235,6 +237,8 @@ export interface RestoreConfig { agentSlackNotifyEnabled?: boolean; /** Sandbox settings (tunnel ports, etc.) resolved from integration settings */ sandboxSettings?: SandboxSettings; + /** Previous logical allocation identity, used by providers supporting ambiguous-create recovery. */ + retireSandboxId?: string | null; /** Multi-repo member list — see CreateSandboxConfig. */ repositories?: SessionRepositoryInfo[]; } @@ -489,6 +493,18 @@ export class SandboxProviderError extends Error { } } +/** A rejected execution; a non-null handle remains a cleanup obligation. */ +export class SandboxLaunchRejectedError extends SandboxProviderError { + constructor( + message: string, + readonly providerObjectId: string | null, + cause?: Error + ) { + super(message, "permanent", cause); + this.name = "SandboxLaunchRejectedError"; + } +} + /** The provider confirmed that the selected prebuilt artifact cannot be restored. */ export class PrebuiltImageUnavailableError extends SandboxProviderError { constructor(message: string, cause?: Error) { diff --git a/packages/control-plane/src/sandbox/providers/modal-backends.test.ts b/packages/control-plane/src/sandbox/providers/modal-backends.test.ts new file mode 100644 index 0000000000..dfccdd876c --- /dev/null +++ b/packages/control-plane/src/sandbox/providers/modal-backends.test.ts @@ -0,0 +1,195 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { createModalClient, type ModalClient } from "../client"; +import { SandboxLaunchRejectedError } from "../provider"; +import { ModalSandboxProvider } from "./modal-provider"; +import { resolveSandboxDashboardUrl } from "../../session/sandbox-access"; + +const config = { + sessionId: "session-1", + sandboxId: "sandbox-1", + repoOwner: null, + repoName: null, + controlPlaneUrl: "https://cp.test", + sandboxAuthToken: "token", + harness: "opencode" as const, + provider: "anthropic", + model: "model", + retireSandboxId: "prior-generation", +}; +const build = { + buildId: "build-1", + scopeKind: "repo" as const, + scopeId: "acme/repo", + repositories: [{ repoOwner: "acme", repoName: "repo", baseBranch: "main" }], + callbackUrl: "https://cp.test/complete", + failureCallbackUrl: "https://cp.test/failed", + callbackToken: "token", + buildExecutionTimeoutSeconds: 60, + providerSessionTimeoutSeconds: 120, + correlation: { trace_id: "trace", request_id: "request" }, +}; + +function fixture(confirmation: unknown) { + const result = { + sandboxId: "sandbox-1", + modalObjectId: "sb-1", + createdAt: 1, + sandboxBackend: confirmation, + }; + const client = { + createSandbox: vi.fn().mockResolvedValue(result), + restoreSandbox: vi.fn().mockResolvedValue(result), + stopSandbox: vi.fn().mockResolvedValue(undefined), + createImageBuildSandbox: vi + .fn() + .mockResolvedValue({ providerSessionId: "sb-1", sandboxBackend: confirmation }), + startImageBuildSandbox: vi.fn().mockResolvedValue(undefined), + snapshotSandbox: vi.fn().mockResolvedValue({ imageId: "im-1", sourceStopped: true }), + }; + return { + client, + provider: new ModalSandboxProvider(client as unknown as ModalClient, "modal-vm"), + }; +} + +afterEach(() => vi.unstubAllGlobals()); + +describe("distinct Modal backend identities", () => { + it("selects the immutable backend on both launch paths without altering generic resources", async () => { + const { provider, client } = fixture("modal-vm"); + const settings = { cpuCores: 3, memoryMib: null }; + await provider.createSandbox({ ...config, sandboxSettings: settings }); + await provider.restoreFromSnapshot({ + ...config, + snapshotImageId: "im-1", + sandboxSettings: settings, + }); + for (const call of [client.createSandbox, client.restoreSandbox]) { + expect(call).toHaveBeenCalledWith( + expect.objectContaining({ + sandboxBackend: "modal-vm", + retireSandboxId: "prior-generation", + sandboxSettings: settings, + }), + undefined + ); + } + expect(provider.name).toBe("modal-vm"); + expect(provider.capabilities.snapshotStopsSandbox).toBe(true); + }); + + it.each([undefined, null, false, "modal", "future-backend", { unexpected: true }])( + "rejects and retires create/restore allocations with confirmation %j", + async (value) => { + const { provider, client } = fixture(value); + await expect(provider.createSandbox(config)).rejects.toThrow("did not confirm"); + await expect( + provider.restoreFromSnapshot({ ...config, snapshotImageId: "im-1" }) + ).rejects.toThrow("did not confirm"); + expect(client.stopSandbox).toHaveBeenCalledTimes(2); + } + ); + + it("retains a rejected session allocation ID when retirement fails", async () => { + const { provider, client } = fixture("modal"); + client.stopSandbox.mockRejectedValue(new Error("unreachable")); + await expect(provider.createSandbox(config)).rejects.toMatchObject({ + name: "SandboxLaunchRejectedError", + providerObjectId: "sb-1", + }); + await expect( + provider.restoreFromSnapshot({ ...config, snapshotImageId: "im-1" }) + ).rejects.toBeInstanceOf(SandboxLaunchRejectedError); + }); + + it.each([undefined, "modal", 42, { bad: true }])( + "binds rejected build handles for cleanup but never starts them (%j)", + async (value) => { + const { provider, client } = fixture(value); + const bind = vi.fn().mockResolvedValue(undefined); + await expect( + provider.triggerImageBuild({ ...build, onProviderSessionCreated: bind }) + ).rejects.toThrow("did not confirm"); + expect(bind).toHaveBeenCalledWith("sb-1"); + expect(client.startImageBuildSandbox).not.toHaveBeenCalled(); + } + ); + + it("starts a confirmed build only after binding", async () => { + const { provider, client } = fixture("modal-vm"); + let bound = false; + client.startImageBuildSandbox.mockImplementation(async () => expect(bound).toBe(true)); + await provider.triggerImageBuild({ + ...build, + onProviderSessionCreated: async () => { + bound = true; + }, + }); + expect(client.startImageBuildSandbox).toHaveBeenCalledOnce(); + }); + + it("preserves an allocation handle when wire confirmation is malformed", async () => { + vi.stubGlobal( + "fetch", + vi.fn().mockResolvedValue( + Response.json({ + success: true, + data: { provider_session_id: "sb-1", sandbox_backend: { invalid: true } }, + }) + ) + ); + const client = createModalClient("test-secret", "workspace"); + const result = await client.createImageBuildSandbox({ ...build, sandboxBackend: "modal-vm" }); + expect(result).toMatchObject({ providerSessionId: "sb-1", sandboxBackend: { invalid: true } }); + }); + + it("serializes backend and predecessor identity on create and restore HTTP requests", async () => { + const fetchMock = vi.fn().mockResolvedValue( + Response.json({ + success: true, + data: { + sandbox_id: "sandbox-1", + modal_object_id: "sb-1", + created_at: 1, + sandbox_backend: "modal-vm", + }, + }) + ); + vi.stubGlobal("fetch", fetchMock); + const client = createModalClient("test-secret", "workspace"); + await client.createSandbox({ ...config, sandboxBackend: "modal-vm" }); + fetchMock.mockResolvedValue( + Response.json({ + success: true, + data: { sandbox_id: "sandbox-1", modal_object_id: "sb-1", sandbox_backend: "modal-vm" }, + }) + ); + await client.restoreSandbox({ ...config, snapshotImageId: "im-1", sandboxBackend: "modal-vm" }); + for (const [, init] of fetchMock.mock.calls) { + expect(JSON.parse(init.body)).toMatchObject({ + sandbox_backend: "modal-vm", + retire_sandbox_id: "prior-generation", + }); + } + }); + + it("requires explicit retirement confirmation for VM session captures", async () => { + const { provider, client } = fixture("modal-vm"); + const input = { providerObjectId: "sb-1", sessionId: "session-1", reason: "checkpoint" }; + await expect(provider.takeSnapshot(input)).resolves.toMatchObject({ + success: true, + sourceStopped: true, + }); + client.snapshotSandbox.mockResolvedValue({ imageId: "im-1", sourceStopped: false }); + await expect(provider.takeSnapshot(input)).rejects.toThrow("did not confirm source retirement"); + }); + + it.each(["modal", "modal-vm"])("keeps dashboard links for %s", (backend) => { + expect( + resolveSandboxDashboardUrl( + { sandboxProvider: backend, modalWorkspace: "acme", modalEnvironment: "main" }, + "sb-1" + ) + ).toContain("sandboxId=sb-1"); + }); +}); diff --git a/packages/control-plane/src/sandbox/providers/modal-provider.test.ts b/packages/control-plane/src/sandbox/providers/modal-provider.test.ts index b7efd04ffd..0d4ba3bc2f 100644 --- a/packages/control-plane/src/sandbox/providers/modal-provider.test.ts +++ b/packages/control-plane/src/sandbox/providers/modal-provider.test.ts @@ -97,7 +97,7 @@ describe("ModalSandboxProvider", () => { try { vi.setSystemTime(new Date("2030-01-01T00:00:00.000Z")); const client = createMockModalClient(); - const provider = new ModalSandboxProvider(client); + const provider = new ModalSandboxProvider(client, "modal"); const created = await provider.createSandbox({ ...testConfig, timeoutSeconds: 1200 }); expect(created.lifetime).toEqual({ kind: "finite", @@ -135,7 +135,7 @@ describe("ModalSandboxProvider", () => { describe("capabilities", () => { it("reports correct capabilities", () => { const client = createMockModalClient(); - const provider = new ModalSandboxProvider(client); + const provider = new ModalSandboxProvider(client, "modal"); expect(provider.name).toBe("modal"); expect(provider.capabilities.supportsSnapshots).toBe(true); @@ -151,7 +151,7 @@ describe("ModalSandboxProvider", () => { throw new Error("fetch failed"); }), }); - const provider = new ModalSandboxProvider(client); + const provider = new ModalSandboxProvider(client, "modal"); await expect(provider.createSandbox(testConfig)).rejects.toThrow(SandboxProviderError); try { @@ -168,7 +168,7 @@ describe("ModalSandboxProvider", () => { throw new Error("connect ETIMEDOUT 192.168.1.1:443"); }), }); - const provider = new ModalSandboxProvider(client); + const provider = new ModalSandboxProvider(client, "modal"); try { await provider.createSandbox(testConfig); @@ -184,7 +184,7 @@ describe("ModalSandboxProvider", () => { throw new Error("read ECONNRESET"); }), }); - const provider = new ModalSandboxProvider(client); + const provider = new ModalSandboxProvider(client, "modal"); try { await provider.createSandbox(testConfig); @@ -200,7 +200,7 @@ describe("ModalSandboxProvider", () => { throw new Error("connect ECONNREFUSED 127.0.0.1:3000"); }), }); - const provider = new ModalSandboxProvider(client); + const provider = new ModalSandboxProvider(client, "modal"); try { await provider.createSandbox(testConfig); @@ -216,7 +216,7 @@ describe("ModalSandboxProvider", () => { throw new Error("Network request failed"); }), }); - const provider = new ModalSandboxProvider(client); + const provider = new ModalSandboxProvider(client, "modal"); try { await provider.createSandbox(testConfig); @@ -232,7 +232,7 @@ describe("ModalSandboxProvider", () => { throw new RequestDeadlineError("Modal", "createSandbox", 30_000); }), }); - const provider = new ModalSandboxProvider(client); + const provider = new ModalSandboxProvider(client, "modal"); try { await provider.createSandbox(testConfig); @@ -248,7 +248,7 @@ describe("ModalSandboxProvider", () => { throw new Error("Modal API error: 502 Bad Gateway"); }), }); - const provider = new ModalSandboxProvider(client); + const provider = new ModalSandboxProvider(client, "modal"); try { await provider.createSandbox(testConfig); @@ -264,7 +264,7 @@ describe("ModalSandboxProvider", () => { throw new Error("Modal API error: 503 Service Unavailable"); }), }); - const provider = new ModalSandboxProvider(client); + const provider = new ModalSandboxProvider(client, "modal"); try { await provider.createSandbox(testConfig); @@ -280,7 +280,7 @@ describe("ModalSandboxProvider", () => { throw new Error("Modal API error: 504 Gateway Timeout"); }), }); - const provider = new ModalSandboxProvider(client); + const provider = new ModalSandboxProvider(client, "modal"); try { await provider.createSandbox(testConfig); @@ -296,7 +296,7 @@ describe("ModalSandboxProvider", () => { throw new Error("upstream bad gateway error"); }), }); - const provider = new ModalSandboxProvider(client); + const provider = new ModalSandboxProvider(client, "modal"); try { await provider.createSandbox(testConfig); @@ -312,7 +312,7 @@ describe("ModalSandboxProvider", () => { throw new Error("service unavailable, try again later"); }), }); - const provider = new ModalSandboxProvider(client); + const provider = new ModalSandboxProvider(client, "modal"); try { await provider.createSandbox(testConfig); @@ -328,7 +328,7 @@ describe("ModalSandboxProvider", () => { throw new Error("gateway timeout while waiting for upstream"); }), }); - const provider = new ModalSandboxProvider(client); + const provider = new ModalSandboxProvider(client, "modal"); try { await provider.createSandbox(testConfig); @@ -346,7 +346,7 @@ describe("ModalSandboxProvider", () => { throw new Error("Modal API error: 401 Unauthorized"); }), }); - const provider = new ModalSandboxProvider(client); + const provider = new ModalSandboxProvider(client, "modal"); try { await provider.createSandbox(testConfig); @@ -362,7 +362,7 @@ describe("ModalSandboxProvider", () => { throw new Error("Modal API error: 403 Forbidden"); }), }); - const provider = new ModalSandboxProvider(client); + const provider = new ModalSandboxProvider(client, "modal"); try { await provider.createSandbox(testConfig); @@ -378,7 +378,7 @@ describe("ModalSandboxProvider", () => { throw new Error("Modal API error: 400 Bad Request - Invalid configuration"); }), }); - const provider = new ModalSandboxProvider(client); + const provider = new ModalSandboxProvider(client, "modal"); try { await provider.createSandbox(testConfig); @@ -394,7 +394,7 @@ describe("ModalSandboxProvider", () => { throw new Error("Modal API error: 422 Unprocessable Entity"); }), }); - const provider = new ModalSandboxProvider(client); + const provider = new ModalSandboxProvider(client, "modal"); try { await provider.createSandbox(testConfig); @@ -410,7 +410,7 @@ describe("ModalSandboxProvider", () => { throw new Error("Invalid repository configuration"); }), }); - const provider = new ModalSandboxProvider(client); + const provider = new ModalSandboxProvider(client, "modal"); try { await provider.createSandbox(testConfig); @@ -426,7 +426,7 @@ describe("ModalSandboxProvider", () => { throw new Error("Quota exceeded: maximum sandboxes reached"); }), }); - const provider = new ModalSandboxProvider(client); + const provider = new ModalSandboxProvider(client, "modal"); try { await provider.createSandbox(testConfig); @@ -442,7 +442,7 @@ describe("ModalSandboxProvider", () => { throw new Error("Something unexpected happened"); }), }); - const provider = new ModalSandboxProvider(client); + const provider = new ModalSandboxProvider(client, "modal"); try { await provider.createSandbox(testConfig); @@ -458,7 +458,7 @@ describe("ModalSandboxProvider", () => { throw "string error"; // Throwing a string, not an Error }), }); - const provider = new ModalSandboxProvider(client); + const provider = new ModalSandboxProvider(client, "modal"); try { await provider.createSandbox(testConfig); @@ -478,7 +478,7 @@ describe("ModalSandboxProvider", () => { throw originalError; }), }); - const provider = new ModalSandboxProvider(client); + const provider = new ModalSandboxProvider(client, "modal"); try { await provider.createSandbox(testConfig); @@ -494,7 +494,7 @@ describe("ModalSandboxProvider", () => { throw new Error("timeout exceeded"); }), }); - const provider = new ModalSandboxProvider(client); + const provider = new ModalSandboxProvider(client, "modal"); try { await provider.createSandbox(testConfig); @@ -520,7 +520,7 @@ describe("ModalSandboxProvider", () => { const client = createMockModalClient({ createSandbox: vi.fn(async () => expectedResult), }); - const provider = new ModalSandboxProvider(client); + const provider = new ModalSandboxProvider(client, "modal"); const result = await provider.createSandbox({ ...testConfig, vncEnabled: true }); @@ -545,7 +545,7 @@ describe("ModalSandboxProvider", () => { }); await expect( - new ModalSandboxProvider(client).createSandbox({ + new ModalSandboxProvider(client, "modal").createSandbox({ ...testConfig, prebuiltImageId: "im-missing", }) @@ -565,7 +565,7 @@ describe("ModalSandboxProvider", () => { }), }); - const error = await new ModalSandboxProvider(client) + const error = await new ModalSandboxProvider(client, "modal") .createSandbox({ ...testConfig, prebuiltImageId: "im-valid" }) .catch((caught: unknown) => caught); @@ -577,7 +577,7 @@ describe("ModalSandboxProvider", () => { describe("image builds", () => { it("binds a created image-build sandbox before starting it", async () => { const client = createMockModalClient(); - const provider = new ModalSandboxProvider(client); + const provider = new ModalSandboxProvider(client, "modal"); const correlation = { request_id: "request-1", trace_id: "trace-1" }; const onProviderSessionCreated = vi.fn(async () => undefined); @@ -599,6 +599,8 @@ describe("ModalSandboxProvider", () => { expect(client.createImageBuildSandbox).toHaveBeenCalledWith( { + sandboxBackend: "modal", + resources: undefined, scopeKind: "repo", scopeId: "acme/repo", buildId: "build-123", @@ -638,7 +640,7 @@ describe("ModalSandboxProvider", () => { throw new ModalApiError("Modal API error: 502 Bad Gateway", 502); }), }); - const provider = new ModalSandboxProvider(client); + const provider = new ModalSandboxProvider(client, "modal"); try { await provider.restoreFromSnapshot({ @@ -666,7 +668,7 @@ describe("ModalSandboxProvider", () => { throw new ModalApiError("Modal API error: 401 Unauthorized", 401); }), }); - const provider = new ModalSandboxProvider(client); + const provider = new ModalSandboxProvider(client, "modal"); try { await provider.restoreFromSnapshot({ @@ -695,7 +697,7 @@ describe("ModalSandboxProvider", () => { throw modalError; }), }); - const provider = new ModalSandboxProvider(client); + const provider = new ModalSandboxProvider(client, "modal"); try { await provider.takeSnapshot({ @@ -718,7 +720,10 @@ describe("ModalSandboxProvider", () => { const snapshotBuildSandbox = vi.fn(async () => ({ imageId: "build-image-123", })); - const provider = new ModalSandboxProvider(createMockModalClient({ snapshotBuildSandbox })); + const provider = new ModalSandboxProvider( + createMockModalClient({ snapshotBuildSandbox }), + "modal" + ); await expect( provider.snapshotImageBuildSandbox({ @@ -742,7 +747,8 @@ describe("ModalSandboxProvider", () => { snapshotBuildSandbox: vi.fn(async () => { throw modalError; }), - }) + }), + "modal" ); await expect( @@ -764,7 +770,7 @@ describe("ModalSandboxProvider", () => { vncPassword: "vnc-pw", })), }); - const provider = new ModalSandboxProvider(client); + const provider = new ModalSandboxProvider(client, "modal"); const result = await provider.restoreFromSnapshot({ snapshotImageId: "img-123", diff --git a/packages/control-plane/src/sandbox/providers/modal-provider.ts b/packages/control-plane/src/sandbox/providers/modal-provider.ts index 8a38631e15..7e2ac05db7 100644 --- a/packages/control-plane/src/sandbox/providers/modal-provider.ts +++ b/packages/control-plane/src/sandbox/providers/modal-provider.ts @@ -6,13 +6,15 @@ */ import { ModalApiError } from "../client"; -import type { ModalClient } from "../client"; +import type { ModalClient, ModalBackend, CreateImageBuildSandboxResponse } from "../client"; +import type { SandboxSettings } from "@open-inspect/shared/types/integrations"; import type { CorrelationContext } from "../../logger"; import { supportsConfigurableSandboxTimeout } from "@open-inspect/shared/types/integrations"; import { DEFAULT_SANDBOX_TIMEOUT_SECONDS, PrebuiltImageUnavailableError, SandboxProviderError, + SandboxLaunchRejectedError, createVncAccess, signalUntilDeadline, type ImageBuildProviderTriggerConfig, @@ -37,6 +39,7 @@ interface StartModalImageBuildConfig { /** Modal extends the shared trigger contract with explicit SCM clone identity. */ export interface ModalImageBuildTriggerConfig extends ImageBuildProviderTriggerConfig { + resources?: Pick; cloneHost?: string; cloneUsername?: string; } @@ -57,6 +60,11 @@ export interface SnapshotModalImageBuildConfig { } export interface ModalImageBuildProvider { + recoverImageBuildSandbox?( + buildId: string, + correlation?: CorrelationContext, + signal?: AbortSignal + ): Promise<{ providerSessionId: string } | null>; triggerImageBuild(config: ModalImageBuildTriggerConfig): Promise; terminateImageBuildSandbox(config: TerminateModalImageBuildConfig): Promise; snapshotImageBuildSandbox(config: SnapshotModalImageBuildConfig): Promise; @@ -88,17 +96,24 @@ export interface ModalImageBuildProvider { * ``` */ export class ModalSandboxProvider implements SandboxProvider, ModalImageBuildProvider { - readonly name = "modal"; + readonly name: ModalBackend; - readonly capabilities: SandboxProviderCapabilities = { - supportsSandboxTimeout: supportsConfigurableSandboxTimeout(this.name), - supportsSnapshots: true, - supportsRestore: true, - supportsPersistentResume: false, - supportsExplicitStop: true, - }; + readonly capabilities: SandboxProviderCapabilities; - constructor(private readonly client: ModalClient) {} + constructor( + private readonly client: ModalClient, + backend: ModalBackend + ) { + this.name = backend; + this.capabilities = { + supportsSandboxTimeout: supportsConfigurableSandboxTimeout(this.name), + supportsSnapshots: true, + snapshotStopsSandbox: backend === "modal-vm", + supportsRestore: true, + supportsPersistentResume: false, + supportsExplicitStop: true, + }; + } /** * Create a new sandbox via Modal API. @@ -129,11 +144,14 @@ export class ModalSandboxProvider implements SandboxProvider, ModalImageBuildPro agentSlackNotifyEnabled: config.agentSlackNotifyEnabled, mcpServers: config.mcpServers, sandboxSettings: config.sandboxSettings, + sandboxBackend: this.name, + retireSandboxId: config.retireSandboxId, repositories: config.repositories, }, config.correlation ); + await this.confirmSessionLaunch(config, result); return { sandboxId: result.sandboxId, providerObjectId: result.modalObjectId, @@ -185,11 +203,14 @@ export class ModalSandboxProvider implements SandboxProvider, ModalImageBuildPro agentSlackNotifyEnabled: config.agentSlackNotifyEnabled, mcpServers: config.mcpServers, sandboxSettings: config.sandboxSettings, + sandboxBackend: this.name, + retireSandboxId: config.retireSandboxId, repositories: config.repositories, }, config.correlation ); + await this.confirmSessionLaunch(config, result); return { success: true, sandboxId: result.sandboxId, @@ -235,9 +256,16 @@ export class ModalSandboxProvider implements SandboxProvider, ModalImageBuildPro config.correlation ); + if (this.name === "modal-vm" && result.sourceStopped !== true) { + throw new SandboxProviderError( + "Modal VM snapshot did not confirm source retirement", + "permanent" + ); + } return { success: true, imageId: result.imageId, + ...(this.name === "modal-vm" ? { sourceStopped: true } : {}), }; } catch (error) { if (error instanceof ModalApiError) { @@ -298,10 +326,12 @@ export class ModalSandboxProvider implements SandboxProvider, ModalImageBuildPro private async createImageBuildSandbox( config: ModalImageBuildTriggerConfig - ): Promise<{ providerSessionId: string }> { + ): Promise { try { return await this.client.createImageBuildSandbox( { + sandboxBackend: this.name, + resources: config.resources, scopeKind: config.scopeKind, scopeId: config.scopeId, buildId: config.buildId, @@ -330,9 +360,60 @@ export class ModalSandboxProvider implements SandboxProvider, ModalImageBuildPro } } + private assertBackend(result: { sandboxBackend?: unknown; legacyDockerEnabled?: unknown }): void { + const legacyStandard = + this.name === "modal" && + result.sandboxBackend === undefined && + (result.legacyDockerEnabled === undefined || result.legacyDockerEnabled === false); + if (result.sandboxBackend === this.name || legacyStandard) return; + throw new SandboxProviderError( + `Modal deployment did not confirm the ${this.name} backend; deploy compatible Modal endpoints`, + "permanent" + ); + } + + private async confirmSessionLaunch( + config: CreateSandboxConfig | RestoreConfig, + result: { modalObjectId?: string; sandboxBackend?: unknown; legacyDockerEnabled?: unknown } + ): Promise { + try { + this.assertBackend(result); + } catch (error) { + if (result.modalObjectId) { + try { + await this.client.stopSandbox( + { providerObjectId: result.modalObjectId, sessionId: config.sessionId }, + config.correlation + ); + } catch (cleanupError) { + throw new SandboxLaunchRejectedError( + "Incompatible Modal allocation could not be retired", + result.modalObjectId, + cleanupError instanceof Error ? cleanupError : undefined + ); + } + } + throw new SandboxLaunchRejectedError( + error instanceof Error ? error.message : "Incompatible Modal allocation", + null, + error instanceof Error ? error : undefined + ); + } + } + + recoverImageBuildSandbox( + buildId: string, + correlation?: CorrelationContext, + signal?: AbortSignal + ): Promise<{ providerSessionId: string } | null> { + return this.client.recoverImageBuildSandbox(buildId, this.name, correlation, signal); + } + async triggerImageBuild(config: ModalImageBuildTriggerConfig): Promise { const created = await this.createImageBuildSandbox(config); + // Persist the handle for cleanup before checking compatibility. Binding does not start work. await config.onProviderSessionCreated(created.providerSessionId); + this.assertBackend(created); await this.startImageBuildSandbox({ buildId: config.buildId, providerSessionId: created.providerSessionId, @@ -392,6 +473,7 @@ export class ModalSandboxProvider implements SandboxProvider, ModalImageBuildPro * Classify an error as transient or permanent for circuit breaker handling. */ private classifyError(message: string, error: unknown): SandboxProviderError { + if (error instanceof SandboxProviderError) return error; if (SandboxProviderError.isTransientNetworkError(error)) { return new SandboxProviderError( `${message}: ${error instanceof Error ? error.message : String(error)}`, @@ -432,6 +514,9 @@ export class ModalSandboxProvider implements SandboxProvider, ModalImageBuildPro * @param client - ModalClient instance for API calls * @returns ModalSandboxProvider instance */ -export function createModalProvider(client: ModalClient): ModalSandboxProvider { - return new ModalSandboxProvider(client); +export function createModalProvider( + client: ModalClient, + backend: ModalBackend +): ModalSandboxProvider { + return new ModalSandboxProvider(client, backend); } diff --git a/packages/control-plane/src/session/components.ts b/packages/control-plane/src/session/components.ts index c888ed50ca..2cf06ba0e7 100644 --- a/packages/control-plane/src/session/components.ts +++ b/packages/control-plane/src/session/components.ts @@ -1055,7 +1055,7 @@ function createLifecycleManager(deps: LifecycleManagerDeps): SandboxLifecycleMan }; const sandboxDashboardUrlBuilder = - sandboxBackend === "modal" + sandboxBackend === "modal" || sandboxBackend === "modal-vm" ? (providerObjectId: string) => resolveSandboxDashboardUrl(sandboxDashboardSettings, providerObjectId) : undefined; diff --git a/packages/control-plane/src/session/sandbox-access.ts b/packages/control-plane/src/session/sandbox-access.ts index 387c5798bd..f8ea09c28d 100644 --- a/packages/control-plane/src/session/sandbox-access.ts +++ b/packages/control-plane/src/session/sandbox-access.ts @@ -80,7 +80,8 @@ export function resolveSandboxDashboardUrl( settings: SandboxDashboardSettings, providerObjectId: string | null | undefined ): string | null { - if (resolveSandboxBackendName(settings.sandboxProvider) !== "modal") return null; + const backend = resolveSandboxBackendName(settings.sandboxProvider); + if (backend !== "modal" && backend !== "modal-vm") return null; return buildModalSandboxDashboardUrl({ workspace: settings.modalWorkspace, modalEnvironment: settings.modalEnvironment, diff --git a/packages/control-plane/src/session/sandbox-repository.test.ts b/packages/control-plane/src/session/sandbox-repository.test.ts index 507e9c8976..a492ab4f0d 100644 --- a/packages/control-plane/src/session/sandbox-repository.test.ts +++ b/packages/control-plane/src/session/sandbox-repository.test.ts @@ -462,6 +462,43 @@ describe("SandboxRepository boot state (SQLite)", () => { return { db, sql, repository, set }; } + describe("rejectProviderStartup", () => { + const generation = { sandboxId: "sb-1", createdAt: 1000 }; + + it.each(["spawning", "connecting", "ready", "failed"])( + "fences %s and persists cleanup responsibility", + (status) => { + const { repository, set } = createSqliteRepository(); + set( + "status = ?, modal_sandbox_id = 'sb-1', fenced = ?, auth_token_hash = 'hash', active_socket_id = 'socket'", + status, + status === "failed" ? 1 : 0 + ); + expect(repository.rejectProviderStartup(generation, "sb-rejected")).toBe(true); + expect(repository.getSandbox()).toMatchObject({ + status: "failed", + fenced: 1, + modal_object_id: "sb-rejected", + auth_token_hash: "", + auth_token: null, + active_socket_id: "", + }); + expect(repository.markSandboxReady(generation)).toBe(false); + } + ); + + it("records confirmed cleanup without retaining an obligation and rejects superseded writes", () => { + const { repository, set } = createSqliteRepository(); + set("status = 'connecting', modal_sandbox_id = 'sb-1', modal_object_id = 'old'"); + expect(repository.rejectProviderStartup(generation, null)).toBe(true); + expect(repository.getSandbox()?.modal_object_id).toBeNull(); + expect(repository.rejectProviderStartup({ ...generation, createdAt: 999 }, "late")).toBe( + false + ); + expect(repository.getSandbox()?.modal_object_id).toBeNull(); + }); + }); + describe("commitProviderStartup", () => { const generation = { sandboxId: "sb-1", createdAt: 1000 }; diff --git a/packages/control-plane/src/session/sandbox-repository.ts b/packages/control-plane/src/session/sandbox-repository.ts index 936b8ba423..3f0af2588d 100644 --- a/packages/control-plane/src/session/sandbox-repository.ts +++ b/packages/control-plane/src/session/sandbox-repository.ts @@ -158,6 +158,24 @@ export class SandboxRepository { return (result.rowsWritten ?? 0) > 0; } + /** Persist cleanup responsibility while permanently revoking a rejected generation. */ + rejectProviderStartup( + generation: { sandboxId: string | null; createdAt: number }, + providerObjectId: string | null + ): boolean { + const result = this.sql.exec( + `UPDATE sandbox SET modal_object_id = ?, status = 'failed', fenced = 1, + auth_token_hash = '', auth_token = NULL, active_socket_id = '' + WHERE id = (SELECT id FROM sandbox LIMIT 1) + AND modal_sandbox_id IS ? AND created_at = ?`, + providerObjectId, + generation.sandboxId, + generation.createdAt + ); + result.toArray(); + return (result.rowsWritten ?? 0) > 0; + } + commitProviderStartup( generation: { sandboxId: string | null; createdAt: number }, providerObjectId: string | null, diff --git a/packages/control-plane/test/integration/modal-backend-builds.test.ts b/packages/control-plane/test/integration/modal-backend-builds.test.ts new file mode 100644 index 0000000000..e91b996054 --- /dev/null +++ b/packages/control-plane/test/integration/modal-backend-builds.test.ts @@ -0,0 +1,110 @@ +import { env } from "cloudflare:test"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { createCloudflareEnv } from "../../src/cloudflare/platform"; +import { ImageBuildStore } from "../../src/db/image-builds"; +import { ModalImageBuildAdapter } from "../../src/image-builds/modal-adapter"; +import type { ImageBuildPlannerPort } from "../../src/image-builds/planner"; +import { ImageBuildSessionCleanup } from "../../src/image-builds/session-cleanup"; +import { ImageBuildWorkflow } from "../../src/image-builds/workflow"; +import type { ModalClient } from "../../src/sandbox/client"; +import { ModalSandboxProvider } from "../../src/sandbox/providers/modal-provider"; +import { cleanD1Tables } from "./cleanup"; +import { seedEnvironment, seedImageRowForScope } from "./image-build-helpers"; + +beforeEach(cleanD1Tables); + +describe("Modal backend images over real D1", () => { + it("partitions ready images and build admission by existing provider identity", async () => { + const scope = { kind: "environment", id: await seedEnvironment() } as const; + const store = new ImageBuildStore(env.DB); + for (const provider of ["modal", "modal-vm"] as const) { + await seedImageRowForScope(scope, { + id: provider, + provider, + status: "ready", + providerImageId: "im-" + provider, + }); + expect( + await store.registerBuild({ + id: "build-" + provider, + scope, + provider, + repositoriesFingerprint: "fp", + callbackTokenHash: "hash", + callbackTokenExpiresAt: Date.now() + 60000, + }) + ).toBe(true); + } + for (const provider of ["modal", "modal-vm"] as const) { + expect(await store.getLatestReadyForSpawn(scope, provider)).toMatchObject({ + provider, + provider_image_id: "im-" + provider, + }); + expect(await store.getActiveBuild(scope, provider)).toMatchObject({ + id: "build-" + provider, + }); + } + }); + + it("never starts an incompatible build and retries durable cleanup after reconstruction", async () => { + const scope = { kind: "environment", id: await seedEnvironment() } as const; + const client = { + createImageBuildSandbox: vi + .fn() + .mockResolvedValue({ providerSessionId: "sb-legacy", sandboxBackend: undefined }), + startImageBuildSandbox: vi.fn(), + terminateImageBuildSandbox: vi.fn().mockRejectedValue(new Error("provider unreachable")), + recoverImageBuildSandbox: vi.fn(), + }; + const factory = { + create: () => + new ModalImageBuildAdapter( + new ModalSandboxProvider(client as unknown as ModalClient, "modal-vm") + ), + }; + const planner: ImageBuildPlannerPort = { + resolveTarget: async () => ({ + kind: "environment", + repositories: [{ repoOwner: "acme", repoName: "web", baseBranch: "main" }], + repositoriesFingerprint: "fp", + }), + createCallbackAuth: async () => ({ + token: "token", + tokenHash: "hash", + expiresAt: Date.now() + 60000, + }), + planBuild: async (input) => ({ + ...input, + repositories: input.target.repositories, + repositoriesFingerprint: input.target.repositoriesFingerprint, + buildTimeoutMs: 60000, + callbackToken: "token", + cloneAuth: { type: "unavailable" }, + }), + }; + const correlation = { trace_id: "trace", request_id: "request" }; + const workflow = new ImageBuildWorkflow( + createCloudflareEnv({ ...env, WORKER_URL: "https://worker.test" }), + new ImageBuildStore(env.DB), + factory, + { provider: "modal-vm", planner } + ); + await expect(workflow.triggerBuild(scope, correlation)).rejects.toThrow( + "Failed to trigger build" + ); + expect(client.startImageBuildSandbox).not.toHaveBeenCalled(); + const restartedStore = new ImageBuildStore(env.DB); + const [pending] = await restartedStore.listSessionCleanup(); + expect(pending).toMatchObject({ + provider: "modal-vm", + provider_session_id: "sb-legacy", + provider_session_cleanup_pending: 1, + }); + expect(await restartedStore.getLatestReadyForSpawn(scope, "modal-vm")).toBeNull(); + client.terminateImageBuildSandbox.mockResolvedValue(undefined); + await expect( + new ImageBuildSessionCleanup(restartedStore, factory).run(pending, correlation) + ).resolves.toBe(true); + expect(await restartedStore.listSessionCleanup()).toEqual([]); + }); +}); diff --git a/packages/modal-infra/README.md b/packages/modal-infra/README.md index 3fdca3d5dd..a8b7b3da0e 100644 --- a/packages/modal-infra/README.md +++ b/packages/modal-infra/README.md @@ -110,6 +110,8 @@ pip install -e ".[dev]" ```bash # Build the dynamic Sandbox image, then deploy the app (recommended) uv run python deploy.py --build-sandbox-image +# Also build and verify the Docker-capable image variant (docs/MODAL_DOCKER.md): +# BUILD_MODAL_VM_IMAGE=true uv run python deploy.py --build-sandbox-image uv run modal deploy deploy.py # Alternative app deployment after the same eager image-build step diff --git a/packages/modal-infra/deploy.py b/packages/modal-infra/deploy.py index b140403e1c..fd21c60452 100644 --- a/packages/modal-infra/deploy.py +++ b/packages/modal-infra/deploy.py @@ -10,6 +10,7 @@ import argparse import json +import os import sys from pathlib import Path @@ -22,16 +23,54 @@ # The eager builder must run before a verified image reference exists. Import # only build modules, without src.__init__ registering deployable functions. from app_config import APP_NAME - from images.base import base_image, base_image_plan, image_reference_path + from images.base import ( + MODAL_VM_EXPERIMENTAL_OPTIONS, + base_image, + base_image_plan, + docker_image, + image_reference_path, + ) + from sandbox.launch_policy import VM_DEFAULT_CPU_CORES, VM_DEFAULT_MEMORY_MIB else: # Modal imports this module to discover the fully registered application. from src.app import app from src.app_config import APP_NAME - from src.images.base import base_image, base_image_plan, image_reference_path + from src.images.base import ( + MODAL_VM_EXPERIMENTAL_OPTIONS, + base_image, + base_image_plan, + docker_image, + image_reference_path, + ) + from src.sandbox.launch_policy import VM_DEFAULT_CPU_CORES, VM_DEFAULT_MEMORY_MIB + +# Resources the Docker image is verified with; the same defaults the control +# plane freezes into a Docker session that configures none. +DOCKER_VERIFICATION_CPU_CORES = VM_DEFAULT_CPU_CORES +DOCKER_VERIFICATION_MEMORY_MIB = VM_DEFAULT_MEMORY_MIB + + +def _verify_image(sandbox: modal.Sandbox, *scripts: tuple[str, ...]) -> None: + for script in scripts: + process = sandbox.exec("/opt/openinspect/python/bin/python", *script, timeout=240) + process.stdout.read() + process.wait() + if process.returncode != 0: + raise RuntimeError(f"Modal image verification failed: {process.stderr.read()}") -def build_sandbox_image() -> None: - """Build the image used by dynamic sandboxes before requests can create them.""" +def _publish_image_record(record: dict[str, str]) -> None: + path = image_reference_path() + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(record) + "\n") + + +def build_sandbox_image(*, with_docker: bool = False) -> None: + """Build the image used by dynamic sandboxes before requests can create them. + + The default image is verified and published first, so a failed Docker + variant never replaces a known-good default reference. + """ if base_image_plan is None: raise RuntimeError("Modal sandbox image build requires a local packed image plan") deployed_app = modal.App.lookup(APP_NAME, create_if_missing=True) @@ -49,16 +88,7 @@ def build_sandbox_image() -> None: timeout=300, ) try: - process = sandbox.exec( - "/opt/openinspect/python/bin/python", - "/app/verify/smoke_test.py", - "verify", - timeout=240, - ) - process.stdout.read() - process.wait() - if process.returncode != 0: - raise RuntimeError(f"Modal image verification failed: {process.stderr.read()}") + _verify_image(sandbox, ("/app/verify/smoke_test.py", "verify")) write_build_result(base_image.object_id) finally: sandbox.terminate() @@ -67,9 +97,36 @@ def build_sandbox_image() -> None: "imageId": base_image.object_id, "buildHash": base_image_plan["buildHash"], } - path = image_reference_path() - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(record) + "\n") + _publish_image_record(record) + if not with_docker: + return + + if docker_image is None: + raise RuntimeError("Docker sandbox image build requires a local packed image plan") + with modal.enable_output(): + docker_image.build(deployed_app) + # Verify on the VM runtime the variant is launched with, running the + # standard smoke suite plus a real daemon, build, run and Compose check. + sandbox = modal.Sandbox.create( + "sleep", + "infinity", + app=deployed_app, + image=modal.Image.from_id(docker_image.object_id), + env=base_image_plan["runtimeEnv"], + timeout=600, + cpu=DOCKER_VERIFICATION_CPU_CORES, + memory=DOCKER_VERIFICATION_MEMORY_MIB, + experimental_options=dict(MODAL_VM_EXPERIMENTAL_OPTIONS), + ) + try: + _verify_image( + sandbox, + ("/app/verify/smoke_test.py", "verify"), + ("/app/verify/docker_smoke.py",), + ) + finally: + sandbox.terminate() + _publish_image_record({**record, "dockerImageId": docker_image.object_id}) def main() -> None: @@ -77,7 +134,8 @@ def main() -> None: parser.add_argument("--build-sandbox-image", action="store_true") args = parser.parse_args() if args.build_sandbox_image: - build_sandbox_image() + # BUILD_MODAL_VM_IMAGE is the same switch Terraform and images/base.py read. + build_sandbox_image(with_docker=os.environ.get("BUILD_MODAL_VM_IMAGE") == "true") if __name__ == "__main__": diff --git a/packages/modal-infra/src/images/base.py b/packages/modal-infra/src/images/base.py index b5c7f09e51..8ec8d28d2a 100644 --- a/packages/modal-infra/src/images/base.py +++ b/packages/modal-infra/src/images/base.py @@ -14,6 +14,11 @@ CACHE_BUSTER = RUNTIME_VERSION IMAGE_ID_ENV = "OPENINSPECT_MODAL_BASE_IMAGE_ID" +# The optional Docker-capable variant: the default image plus one install phase. +# Provisioned separately; absent until an operator builds and verifies it. +DOCKER_IMAGE_ID_ENV = "OPENINSPECT_MODAL_DOCKER_IMAGE_ID" +# The Modal option that launches a Docker-capable VM instead of a gVisor sandbox. +MODAL_VM_EXPERIMENTAL_OPTIONS: dict[str, bool] = {"vm_runtime": True} def local_image_plan() -> tuple[Path, dict[str, Any]]: @@ -36,7 +41,11 @@ def deployed_image_environment() -> dict[str, str]: image_id = os.environ.get(IMAGE_ID_ENV) if not image_id: raise RuntimeError("Deployed Modal function is missing its verified sandbox image ID") - return {IMAGE_ID_ENV: image_id} + environment = {IMAGE_ID_ENV: image_id} + docker_image_id = os.environ.get(DOCKER_IMAGE_ID_ENV) + if docker_image_id: + environment[DOCKER_IMAGE_ID_ENV] = docker_image_id + return environment path = image_reference_path() if not path.is_file(): raise RuntimeError("Build the Modal sandbox image before deploying functions") @@ -47,7 +56,15 @@ def deployed_image_environment() -> dict[str, str]: image_id = record.get("imageId") if not isinstance(image_id, str) or not image_id.strip(): raise RuntimeError("Built Modal image record is missing its verified sandbox image ID") - return {IMAGE_ID_ENV: image_id} + environment = {IMAGE_ID_ENV: image_id} + docker_image_id = record.get("dockerImageId") + if docker_image_id is not None: + if not isinstance(docker_image_id, str) or not docker_image_id.strip(): + raise RuntimeError("Built Modal image record has an invalid Docker image ID") + environment[DOCKER_IMAGE_ID_ENV] = docker_image_id + if os.environ.get("BUILD_MODAL_VM_IMAGE") == "true" and DOCKER_IMAGE_ID_ENV not in environment: + raise RuntimeError("Build and verify the Docker sandbox image before deploying functions") + return environment def _define_image() -> tuple[modal.Image, dict[str, Any] | None]: @@ -68,3 +85,16 @@ def _define_image() -> tuple[modal.Image, dict[str, Any] | None]: base_image, base_image_plan = _define_image() + + +def _define_docker_image() -> modal.Image | None: + if not modal.is_local(): + image_id = os.environ.get(DOCKER_IMAGE_ID_ENV) + return modal.Image.from_id(image_id) if image_id else None + # Same bundle, one extra phase; the default image layers stay untouched. + return base_image.run_commands( + "bash /tmp/openinspect-image/packages/sandbox-images/install/install.sh docker" + ) + + +docker_image = _define_docker_image() diff --git a/packages/modal-infra/src/sandbox/build_session.py b/packages/modal-infra/src/sandbox/build_session.py index 2dfd467be3..c01c0d4b8f 100644 --- a/packages/modal-infra/src/sandbox/build_session.py +++ b/packages/modal-infra/src/sandbox/build_session.py @@ -2,6 +2,7 @@ import json import time +from dataclasses import dataclass from typing import cast import modal @@ -22,7 +23,16 @@ ) from ..app import app +from ..app_config import APP_NAME from ..images.base import base_image +from .launch_policy import ( + ModalBackend, + _identity_digest, + docker_base_image, + docker_runtime_env, + launch_kwargs, + parse_launch, +) from .manager import SNAPSHOT_FILESYSTEM_TIMEOUT_SECONDS from .vcs_env import inject_vcs_env_vars @@ -52,6 +62,12 @@ class BuildSessionNotFoundError(LookupError): """The requested provider session is absent or bound to another build.""" +@dataclass(frozen=True) +class BuildSessionLaunch: + provider_session_id: str + sandbox_backend: ModalBackend + + class ModalBuildSessionService: """Own the identity-bound lifecycle of one Modal image-build sandbox.""" @@ -70,8 +86,13 @@ async def create( user_env_vars: dict[str, str] | None = None, build_execution_timeout_seconds: int = DEFAULT_BUILD_TIMEOUT_SECONDS, timeout_seconds: int = DEFAULT_BUILD_TIMEOUT_SECONDS, - ) -> str: + sandbox_settings: dict | None = None, + sandbox_backend: ModalBackend = "modal", + ) -> BuildSessionLaunch: start_time = time.time() + docker = parse_launch(sandbox_backend, sandbox_settings) + if sandbox_backend == "modal": + docker = parse_launch(sandbox_backend, None) # Preserve standard build sizing. primary = repositories[0] env_vars = dict(user_env_vars or {}) for name in RESERVED_USER_ENV_KEYS: @@ -93,6 +114,7 @@ async def create( BUILD_ID_ENV: build_id, CALLBACK_URL_ENV: callback_url, FAILURE_CALLBACK_URL_ENV: failure_callback_url, + **docker_runtime_env(docker), } ) inject_vcs_env_vars( @@ -109,18 +131,31 @@ async def create( "openinspect_scope_kind": scope_kind, "openinspect_scope_id": scope_id, LAUNCH_PROTOCOL_TAG: MODAL_IMAGE_BUILD_START_PROTOCOL, + "openinspect_backend": sandbox_backend, } - sandbox = await modal.Sandbox.create.aio( - *command, - image=base_image, - app=app, - secrets=[], - timeout=timeout_seconds, - workdir="/workspace", - env=cast("dict[str, str | None]", env_vars), - tags=tags, - ) + name = self._allocation_name(build_id, sandbox_backend) + sandbox = await self._find(build_id, sandbox_backend) + if sandbox is not None and await sandbox.get_tags.aio() != tags: + raise RuntimeError("Build allocation ownership mismatch") + if sandbox is None: + try: + sandbox = await modal.Sandbox.create.aio( + *command, + image=docker_base_image() if docker.enabled else base_image, + app=app, + secrets=[], + timeout=timeout_seconds, + workdir="/workspace", + env=cast("dict[str, str | None]", env_vars), + tags=tags, + name=name, + **launch_kwargs(docker), + ) + except modal.exception.AlreadyExistsError: + sandbox = await self._find(build_id, sandbox_backend) + if sandbox is None or await sandbox.get_tags.aio() != tags: + raise RuntimeError("Build allocation ownership mismatch") from None log.info( "sandbox.create_build", build_id=build_id, @@ -130,7 +165,34 @@ async def create( duration_ms=int((time.time() - start_time) * 1000), outcome="success", ) - return sandbox.object_id + return BuildSessionLaunch( + provider_session_id=sandbox.object_id, sandbox_backend=docker.backend + ) + + @staticmethod + def _allocation_name(build_id: str, backend: ModalBackend) -> str: + return "oi-build-" + _identity_digest(backend, build_id)[:40] + + @classmethod + async def _find(cls, build_id: str, backend: ModalBackend) -> modal.Sandbox | None: + try: + sandbox = await modal.Sandbox.from_name.aio( + APP_NAME, cls._allocation_name(build_id, backend) + ) + except modal.exception.NotFoundError: + return None + tags = await sandbox.get_tags.aio() + if ( + tags.get("openinspect_kind") != "image-build" + or tags.get("openinspect_build_id") != build_id + or tags.get("openinspect_backend") != backend + ): + raise RuntimeError("Build allocation ownership mismatch") + return sandbox + + async def recover(self, *, build_id: str, sandbox_backend: ModalBackend) -> str | None: + sandbox = await self._find(build_id, sandbox_backend) + return sandbox.object_id if sandbox is not None else None async def start( self, diff --git a/packages/modal-infra/src/sandbox/launch_policy.py b/packages/modal-infra/src/sandbox/launch_policy.py new file mode 100644 index 0000000000..05c753fd14 --- /dev/null +++ b/packages/modal-infra/src/sandbox/launch_policy.py @@ -0,0 +1,119 @@ +"""Modal compute offerings: launch resources, runtime options, and allocation identity.""" + +from __future__ import annotations + +import hashlib +import math +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Literal + +from sandbox_runtime.constants import DOCKER_ENABLED_ENV_VAR + +if TYPE_CHECKING: + import modal + +ALLOCATION_NAME_PREFIX = "oi-" +ALLOCATION_KIND_TAG = "openinspect_kind" +ALLOCATION_SESSION_TAG = "openinspect_session_id" +ALLOCATION_SANDBOX_TAG = "openinspect_sandbox_id" +ALLOCATION_BACKEND_TAG = "openinspect_backend" +ModalBackend = Literal["modal", "modal-vm"] +VM_DEFAULT_CPU_CORES = 2 +VM_DEFAULT_MEMORY_MIB = 4096 + + +class InvalidDockerSettingsError(ValueError): + """The Docker-sensitive settings are malformed or contradictory.""" + + +class DockerImageUnavailableError(RuntimeError): + """The deployment has no verified Docker-capable sandbox image.""" + + +@dataclass(frozen=True) +class ModalLaunch: + backend: ModalBackend + cpu_cores: float | None = None + memory_mib: int | None = None + + @property + def enabled(self) -> bool: + return self.backend == "modal-vm" + + +def parse_launch(backend: ModalBackend, settings: dict[str, Any] | None) -> ModalLaunch: + """Select an offering independently of generic resource settings.""" + if backend not in ("modal", "modal-vm"): + raise InvalidDockerSettingsError("Unknown Modal sandbox backend") + settings = settings or {} + if "dockerEnabled" in settings: + raise InvalidDockerSettingsError( + "dockerEnabled was removed; select SANDBOX_PROVIDER=modal-vm" + ) + cpu_cores = settings.get("cpuCores") + memory_mib = settings.get("memoryMib") + if backend == "modal-vm": + cpu_cores = VM_DEFAULT_CPU_CORES if cpu_cores is None else cpu_cores + memory_mib = VM_DEFAULT_MEMORY_MIB if memory_mib is None else memory_mib + if cpu_cores is not None and ( + isinstance(cpu_cores, bool) + or not isinstance(cpu_cores, int | float) + or not math.isfinite(cpu_cores) + or cpu_cores <= 0 + ): + raise InvalidDockerSettingsError("cpuCores must be positive and finite") + if memory_mib is not None and ( + isinstance(memory_mib, bool) or not isinstance(memory_mib, int) or memory_mib <= 0 + ): + raise InvalidDockerSettingsError("memoryMib must be a positive integer") + return ModalLaunch(backend=backend, cpu_cores=cpu_cores, memory_mib=memory_mib) + + +def docker_base_image() -> modal.Image: + """The verified Docker-capable base image; never the default image.""" + from ..images.base import docker_image + + if docker_image is None: + raise DockerImageUnavailableError("Docker sandbox image is not provisioned") + return docker_image + + +def launch_kwargs(launch: ModalLaunch) -> dict[str, Any]: + """One mapping for outer allocation resources and VM runtime selection.""" + result: dict[str, Any] = {} + if launch.enabled: + result["experimental_options"] = {"vm_runtime": True} + if launch.cpu_cores is not None: + result["cpu"] = launch.cpu_cores + if launch.memory_mib is not None: + result["memory"] = launch.memory_mib + return result + + +def docker_runtime_env(launch: ModalLaunch) -> dict[str, str]: + """The trusted runtime signal, always set explicitly by the provider.""" + return {DOCKER_ENABLED_ENV_VAR: "true" if launch.enabled else "false"} + + +def _identity_digest(*parts: str) -> str: + return hashlib.sha256("\n".join(parts).encode()).hexdigest() + + +def docker_allocation_name(session_id: str, sandbox_id: str) -> str: + """Deterministic, Modal-safe sandbox name for one control-plane generation. + + `sandbox_id` already embeds the generation timestamp, so the pair names + exactly one launch attempt. Modal names are limited to 64 chars of + `[A-Za-z0-9._-]`, which control-plane identifiers do not satisfy directly. + """ + return ALLOCATION_NAME_PREFIX + _identity_digest("modal-vm", session_id, sandbox_id)[:40] + + +def docker_allocation_tags(session_id: str, sandbox_id: str) -> dict[str, str]: + """Ownership tags a found allocation must match exactly before adoption or retirement.""" + return { + ALLOCATION_KIND_TAG: "session", + ALLOCATION_SESSION_TAG: _identity_digest(session_id)[:48], + ALLOCATION_SANDBOX_TAG: _identity_digest(sandbox_id)[:48], + ALLOCATION_BACKEND_TAG: "modal-vm", + } diff --git a/packages/modal-infra/src/sandbox/manager.py b/packages/modal-infra/src/sandbox/manager.py index 45aaacfd05..155fce3c30 100644 --- a/packages/modal-infra/src/sandbox/manager.py +++ b/packages/modal-infra/src/sandbox/manager.py @@ -21,6 +21,7 @@ CODE_SERVER_PORT, CODE_SERVER_PORT_ENV_VAR, DEFAULT_SANDBOX_TIMEOUT_SECONDS, + DOCKER_ENABLED_ENV_VAR, EXPECTED_TUNNEL_PORTS_ENV_VAR, NOVNC_PORT, NOVNC_PORT_ENV_VAR, @@ -37,12 +38,23 @@ from sandbox_runtime.types import SandboxStatus, SessionConfig from ..app import app, llm_secrets +from ..app_config import APP_NAME from ..images.base import base_image +from .launch_policy import ( + ModalBackend, + docker_allocation_name, + docker_allocation_tags, + docker_base_image, + docker_runtime_env, + launch_kwargs, + parse_launch, +) from .vcs_env import inject_vcs_env_vars log = get_logger("manager") SNAPSHOT_FILESYSTEM_TIMEOUT_SECONDS = 300 +ACCESS_PASSWORD_READ_TIMEOUT_SECONDS = 30 MAX_TUNNEL_PORTS = 10 DEFAULT_VNC_ENABLED = False _RESERVED_LAUNCH_ENV_VARS = { @@ -55,6 +67,7 @@ "SESSION_CONFIG", VNC_PASSWORD_ENV_VAR, NOVNC_PORT_ENV_VAR, + DOCKER_ENABLED_ENV_VAR, } @@ -70,27 +83,32 @@ def _has_repository(repo_owner: str | None, repo_name: str | None) -> bool: return has_owner -def _resource_kwargs(settings: dict[str, Any] | None) -> dict: - """Map sandbox settings to Modal resource kwargs. - - `cpuCores` -> Modal `cpu` (cores, fractional allowed), `memoryMib` -> Modal - `memory` (MiB). The control plane owns normalization; this only maps - already-normalized settings into provider-specific argument names. - """ - if not settings: - return {} - - kwargs: dict = {} - - cpu_cores = settings.get("cpuCores") - if cpu_cores is not None: - kwargs["cpu"] = float(cpu_cores) +async def _create_sandbox( + create_kwargs: dict[str, Any], *, repository_image: bool +) -> modal.Sandbox: + """The one `Sandbox.create` call; only its own NotFound means the image is gone.""" + try: + return await modal.Sandbox.create.aio( + "python", + "-m", + "sandbox_runtime.entrypoint", + **create_kwargs, + ) + except modal.exception.NotFoundError as e: + if repository_image: + raise RepositoryImageUnavailableError("repository image is unavailable") from e + raise - memory_mib = settings.get("memoryMib") - if memory_mib is not None: - kwargs["memory"] = memory_mib - return kwargs +def _session_identity(session_config: SessionConfig | dict[str, Any] | None) -> str: + """The control-plane session id carried in the launch's session config.""" + if isinstance(session_config, dict): + session_id = session_config.get("session_id") + elif session_config is not None: + session_id = session_config.session_id + else: + session_id = None + return session_id if isinstance(session_id, str) else "" @dataclass @@ -99,6 +117,7 @@ class SandboxConfig: repo_owner: str | None repo_name: str | None + sandbox_backend: ModalBackend = "modal" sandbox_id: str | None = None # Expected sandbox ID from control plane session_config: SessionConfig | dict[str, Any] | None = None control_plane_url: str = "" @@ -115,6 +134,10 @@ class SandboxConfig: settings: dict[str, Any] | None = ( None # Sandbox settings (tunnelPorts, etc.) from control plane ) + # A previous generation's sandbox id whose Docker VM may still be running + # after an ambiguous create (the control plane lost the response). Only + # Docker launches act on it; the named allocation is retired if owned. + retire_sandbox_id: str | None = None @dataclass @@ -133,6 +156,7 @@ class SandboxHandle: vnc_password: str | None = None ttyd_url: str | None = None # proxy tunnel URL (not ttyd directly) tunnel_urls: dict[int, str] | None = None # port -> tunnel URL mapping for extra ports + sandbox_backend: ModalBackend = "modal" @dataclass(frozen=True) @@ -368,6 +392,7 @@ async def _launch_sandbox(self, spec: _SandboxLaunchSpec) -> SandboxHandle: ) sandbox_id = f"sandbox-{sandbox_name}-{int(time.time() * 1000)}" + docker = parse_launch(config.sandbox_backend, config.settings) env_vars = { key: value for key, value in (config.user_env_vars or {}).items() @@ -382,6 +407,7 @@ async def _launch_sandbox(self, spec: _SandboxLaunchSpec) -> SandboxHandle: SANDBOX_TIMEOUT_ENV_VAR: str(config.timeout_seconds), "REPO_OWNER": config.repo_owner or "", "REPO_NAME": config.repo_name or "", + **docker_runtime_env(docker), } ) @@ -389,7 +415,7 @@ async def _launch_sandbox(self, spec: _SandboxLaunchSpec) -> SandboxHandle: include_github_cli_aliases = False snapshot_id: str | None = None if isinstance(spec.source, _BaseImageSource): - image = base_image + image = docker_base_image() if docker.enabled else base_image elif isinstance(spec.source, _RepositoryImageSource): try: image = modal.Image.from_id(spec.source.image_id) @@ -460,22 +486,30 @@ async def _launch_sandbox(self, spec: _SandboxLaunchSpec) -> SandboxHandle: "timeout": config.timeout_seconds, "workdir": "/workspace", "env": env_vars, - **_resource_kwargs(config.settings), + **launch_kwargs(docker), } if exposed_ports: create_kwargs["encrypted_ports"] = exposed_ports - try: - sandbox = await modal.Sandbox.create.aio( - "python", - "-m", - "sandbox_runtime.entrypoint", - **create_kwargs, + repository_image = isinstance(spec.source, _RepositoryImageSource) + if docker.enabled: + sandbox, adopted = await self._launch_docker_sandbox( + session_id=_session_identity(config.session_config), + sandbox_id=sandbox_id, + retire_sandbox_id=config.retire_sandbox_id, + create_kwargs=create_kwargs, + repository_image=repository_image, ) - except modal.exception.NotFoundError as e: - if isinstance(spec.source, _RepositoryImageSource): - raise RepositoryImageUnavailableError("repository image is unavailable") from e - raise + if adopted: + passwords = await self._read_access_passwords( + sandbox, + code_server_enabled=config.code_server_enabled, + vnc_enabled=config.vnc_enabled, + ) + code_server_password = passwords.get("CODE_SERVER_PASSWORD") + vnc_password = passwords.get(VNC_PASSWORD_ENV_VAR) + else: + sandbox = await _create_sandbox(create_kwargs, repository_image=repository_image) modal_object_id = sandbox.object_id ( code_server_url, @@ -507,6 +541,108 @@ async def _launch_sandbox(self, spec: _SandboxLaunchSpec) -> SandboxHandle: vnc_password=vnc_password, ttyd_url=ttyd_url, tunnel_urls=extra_tunnel_urls, + sandbox_backend=docker.backend, + ) + + async def _launch_docker_sandbox( + self, + *, + session_id: str, + sandbox_id: str, + retire_sandbox_id: str | None, + create_kwargs: dict[str, Any], + repository_image: bool, + ) -> tuple[modal.Sandbox, bool]: + """Create a Docker VM under a deterministic name, adopting an existing one. + + VM creation can outlive the control plane's HTTP request. Naming each + generation's allocation lets a retried create adopt the sandbox Modal + already made instead of starting a second VM, and lets the next + generation retire a predecessor whose object id was never learned. + """ + if retire_sandbox_id: + await self._retire_docker_allocation(session_id, retire_sandbox_id) + name = docker_allocation_name(session_id, sandbox_id) + tags = docker_allocation_tags(session_id, sandbox_id) + existing = await self._find_owned_docker_allocation(name, tags) + if existing is None: + try: + sandbox = await _create_sandbox( + {**create_kwargs, "name": name, "tags": tags}, + repository_image=repository_image, + ) + return sandbox, False + except modal.exception.AlreadyExistsError: + existing = await self._find_owned_docker_allocation(name, tags) + if existing is None: + raise + log.info( + "sandbox.docker_allocation_adopted", + sandbox_id=sandbox_id, + modal_object_id=existing.object_id, + ) + return existing, True + + @staticmethod + async def _read_access_passwords( + sandbox: modal.Sandbox, *, code_server_enabled: bool, vnc_enabled: bool + ) -> dict[str, str]: + """Recover only enabled service credentials from the owned VM's launch environment.""" + keys = [] + if code_server_enabled: + keys.append("CODE_SERVER_PASSWORD") + if vnc_enabled: + keys.append(VNC_PASSWORD_ENV_VAR) + if not keys: + return {} + process = await sandbox.exec.aio( + "python", + "-I", + "-c", + "import json, os, sys; print(json.dumps({k: os.environ.get(k) for k in sys.argv[1:]}))", + *keys, + timeout=ACCESS_PASSWORD_READ_TIMEOUT_SECONDS, + ) + output = await process.stdout.read.aio() + if await process.wait.aio() != 0: + raise RuntimeError("Could not recover adopted sandbox access credentials") + try: + passwords = json.loads(output) + except ValueError: + raise RuntimeError("Could not recover adopted sandbox access credentials") from None + if not isinstance(passwords, dict) or any( + not isinstance(passwords.get(key), str) or not passwords[key] for key in keys + ): + raise RuntimeError("Could not recover adopted sandbox access credentials") + return {key: passwords[key] for key in keys} + + @staticmethod + async def _find_owned_docker_allocation( + name: str, tags: dict[str, str] + ) -> modal.Sandbox | None: + try: + sandbox = await modal.Sandbox.from_name.aio(APP_NAME, name) + except modal.exception.NotFoundError: + return None + if await sandbox.get_tags.aio() != tags: + raise RuntimeError("Docker sandbox allocation ownership mismatch") + return sandbox + + async def _retire_docker_allocation(self, session_id: str, sandbox_id: str) -> None: + """Terminate a prior generation's named VM, only when its ownership tags match.""" + name = docker_allocation_name(session_id, sandbox_id) + try: + sandbox = await modal.Sandbox.from_name.aio(APP_NAME, name) + except modal.exception.NotFoundError: + return + if await sandbox.get_tags.aio() != docker_allocation_tags(session_id, sandbox_id): + log.warn("sandbox.docker_allocation_retire_mismatch", sandbox_id=sandbox_id) + return + await sandbox.terminate.aio(wait=True) + log.info( + "sandbox.docker_allocation_retired", + sandbox_id=sandbox_id, + modal_object_id=sandbox.object_id, ) async def create_sandbox( @@ -584,6 +720,23 @@ async def take_snapshot( snapshot_timeout_seconds = min(int(timeout_seconds), SNAPSHOT_FILESYSTEM_TIMEOUT_SECONDS) if snapshot_timeout_seconds <= 0: raise TimeoutError("Insufficient time remains for a filesystem snapshot") + if handle.sandbox_backend == "modal-vm": + preparation_started = time.monotonic() + probe = await handle.modal_sandbox.exec.aio( + "python", + "-m", + "sandbox_runtime.docker_control", + "prepare", + timeout=min(snapshot_timeout_seconds, 45), + ) + if await probe.wait.aio() != 0: + raise RuntimeError("Modal VM Docker shutdown preparation was not confirmed") + snapshot_timeout_seconds = min( + int(timeout_seconds - (time.monotonic() - preparation_started)), + SNAPSHOT_FILESYSTEM_TIMEOUT_SECONDS, + ) + if snapshot_timeout_seconds <= 0: + raise TimeoutError("Snapshot deadline expired during Docker preparation") image = await handle.modal_sandbox.snapshot_filesystem.aio(timeout=snapshot_timeout_seconds) # The image object_id is the unique identifier for this snapshot @@ -624,7 +777,12 @@ async def get_sandbox_by_id(self, sandbox_id: str) -> SandboxHandle | None: """ try: modal_sandbox = await modal.Sandbox.from_id.aio(sandbox_id) + tags = await modal_sandbox.get_tags.aio() + backend = tags.get("openinspect_backend", "modal") + if backend not in ("modal", "modal-vm"): + raise ValueError("Unknown sandbox backend tag") return SandboxHandle( + sandbox_backend="modal-vm" if backend == "modal-vm" else "modal", sandbox_id=sandbox_id, modal_sandbox=modal_sandbox, status=SandboxStatus.READY, # Assume ready if we can retrieve it @@ -648,6 +806,8 @@ async def restore_from_snapshot( vnc_enabled: bool = DEFAULT_VNC_ENABLED, agent_slack_notify_enabled: bool = False, settings: dict[str, Any] | None = None, + retire_sandbox_id: str | None = None, + sandbox_backend: ModalBackend = "modal", ) -> SandboxHandle: """ Create a new sandbox from a filesystem snapshot Image. @@ -698,7 +858,9 @@ async def restore_from_snapshot( code_server_enabled=code_server_enabled, vnc_enabled=vnc_enabled, agent_slack_notify_enabled=agent_slack_notify_enabled, + retire_sandbox_id=retire_sandbox_id, settings=settings, + sandbox_backend=sandbox_backend, ), source=_SnapshotImageSource( image_id=snapshot_image_id, diff --git a/packages/modal-infra/src/web_api.py b/packages/modal-infra/src/web_api.py index a6c59b6029..3c754f5a4c 100644 --- a/packages/modal-infra/src/web_api.py +++ b/packages/modal-infra/src/web_api.py @@ -12,6 +12,7 @@ """ import asyncio +import math import time from collections.abc import AsyncIterator from contextlib import asynccontextmanager @@ -36,6 +37,11 @@ ) from .clone_token import resolve_clone_token from .log_config import configure_logging, get_logger +from .sandbox.launch_policy import ( + DockerImageUnavailableError, + InvalidDockerSettingsError, + ModalBackend, +) configure_logging() log = get_logger("web_api") @@ -76,6 +82,13 @@ class CreateBuildSandboxRequest(_ModalRequestModel): user_env_vars: dict[str, str] | None = None build_execution_timeout_seconds: int | None = None provider_session_timeout_seconds: int | None = None + sandbox_settings: dict[str, Any] | None = None + sandbox_backend: ModalBackend = "modal" + + +class RecoverBuildSandboxRequest(_ModalRequestModel): + build_id: NonEmptyString + sandbox_backend: ModalBackend class StartBuildSandboxRequest(_ModalRequestModel): @@ -136,6 +149,8 @@ class CreateSandboxRequest(_RepositoryContextModel): vnc_enabled: bool | None = None agent_slack_notify_enabled: bool = False sandbox_settings: dict[str, Any] | None = None + sandbox_backend: ModalBackend = "modal" + retire_sandbox_id: str | None = None class RestoreSessionConfigRequest(_RepositoryContextModel): @@ -168,6 +183,8 @@ class RestoreSandboxRequest(_ModalRequestModel): vnc_enabled: bool | None = None agent_slack_notify_enabled: bool = False sandbox_settings: dict[str, Any] | None = None + sandbox_backend: ModalBackend = "modal" + retire_sandbox_id: str | None = None @dataclass @@ -208,6 +225,16 @@ async def _execute_endpoint( execution.http_status = e.status_code execution.outcome = "error" raise + except InvalidDockerSettingsError as e: + execution.http_status = 400 + execution.outcome = "error" + raise HTTPException(status_code=400, detail=str(e)) from e + except DockerImageUnavailableError as e: + # Not provisioned on this deployment: a permanent, actionable failure + # for the request, never a reason to launch the default sandbox. + execution.http_status = 501 + execution.outcome = "error" + raise HTTPException(status_code=501, detail="docker_not_available") from e except Exception as e: execution.http_status = 500 execution.outcome = "error" @@ -429,11 +456,13 @@ async def api_create_sandbox( ), agent_slack_notify_enabled=parsed_request.agent_slack_notify_enabled, settings=parsed_request.sandbox_settings or None, + sandbox_backend=parsed_request.sandbox_backend, timeout_seconds=( parsed_request.timeout_seconds if parsed_request.timeout_seconds is not None else DEFAULT_SANDBOX_TIMEOUT_SECONDS ), + retire_sandbox_id=parsed_request.retire_sandbox_id or None, ) try: @@ -454,6 +483,7 @@ async def api_create_sandbox( "vnc_password": handle.vnc_password, "ttyd_url": handle.ttyd_url, "tunnel_urls": handle.tunnel_urls, + "sandbox_backend": handle.sandbox_backend, }, } @@ -511,7 +541,7 @@ async def api_snapshot_sandbox( if not sandbox_id: raise HTTPException(status_code=400, detail="sandbox_id is required") - from .sandbox.manager import SandboxManager + from .sandbox.manager import SNAPSHOT_FILESYSTEM_TIMEOUT_SECONDS, SandboxManager manager = SandboxManager() @@ -520,22 +550,34 @@ async def api_snapshot_sandbox( raise HTTPException(status_code=404, detail=f"Sandbox not found: {sandbox_id}") deadline_at_ms = request.get("deadline_at_ms") + timeout_seconds = SNAPSHOT_FILESYSTEM_TIMEOUT_SECONDS if deadline_at_ms is not None: - if isinstance(deadline_at_ms, bool) or not isinstance(deadline_at_ms, (int, float)): + if ( + isinstance(deadline_at_ms, bool) + or not isinstance(deadline_at_ms, (int, float)) + or not math.isfinite(deadline_at_ms) + ): raise HTTPException(status_code=400, detail="deadline_at_ms must be a number") timeout_seconds = (deadline_at_ms / 1000) - time.time() if timeout_seconds <= 0: raise HTTPException(status_code=408, detail="snapshot deadline expired") - try: - image_id = await manager.take_snapshot(handle, timeout_seconds=timeout_seconds) - except (TimeoutError, ModalTimeoutError) as exc: - raise HTTPException(status_code=408, detail="snapshot deadline expired") from exc - else: - image_id = await manager.take_snapshot(handle) - + source_stopped = handle.sandbox_backend == "modal-vm" + try: + # Include Docker preparation, capture and confirmed retirement in one budget. + # Timeout is an unknown outcome, never a successful source-stopped receipt. + async with asyncio.timeout(timeout_seconds): + if deadline_at_ms is None: + image_id = await manager.take_snapshot(handle) + else: + image_id = await manager.take_snapshot(handle, timeout_seconds=timeout_seconds) + if source_stopped: + await handle.modal_sandbox.terminate.aio(wait=True) + except (TimeoutError, ModalTimeoutError) as exc: + raise HTTPException(status_code=408, detail="snapshot deadline expired") from exc return { "success": True, "data": { + "source_stopped": source_stopped, "image_id": image_id, "sandbox_id": sandbox_id, }, @@ -702,6 +744,8 @@ async def api_restore_sandbox( ), agent_slack_notify_enabled=parsed_request.agent_slack_notify_enabled, settings=parsed_request.sandbox_settings or None, + sandbox_backend=parsed_request.sandbox_backend, + retire_sandbox_id=parsed_request.retire_sandbox_id or None, ) return { @@ -716,6 +760,7 @@ async def api_restore_sandbox( "vnc_password": handle.vnc_password, "ttyd_url": handle.ttyd_url, "tunnel_urls": handle.tunnel_urls, + "sandbox_backend": handle.sandbox_backend, }, } @@ -777,7 +822,7 @@ async def api_create_build_sandbox( status_code=400, detail="callback URLs must target the control plane" ) - provider_session_id = await ModalBuildSessionService().create( + launch = await ModalBuildSessionService().create( build_id=build_id, scope_kind=scope_kind, scope_id=scope_id, @@ -790,11 +835,16 @@ async def api_create_build_sandbox( user_env_vars=parsed_request.user_env_vars or None, build_execution_timeout_seconds=build_execution_timeout_seconds, timeout_seconds=provider_session_timeout_seconds, + sandbox_settings=parsed_request.sandbox_settings or None, + sandbox_backend=parsed_request.sandbox_backend, ) - execution.log_fields["sandbox_id"] = provider_session_id + execution.log_fields["sandbox_id"] = launch.provider_session_id return { "success": True, - "data": {"provider_session_id": provider_session_id}, + "data": { + "provider_session_id": launch.provider_session_id, + "sandbox_backend": launch.sandbox_backend, + }, } @@ -899,3 +949,28 @@ def _validated_build_repositories( } for repository in repositories ] + + +@app.function(image=function_image, secrets=[internal_api_secret]) +@fastapi_endpoint(method="POST") +async def api_recover_build_sandbox( + request: dict[str, object], + authorization: str | None = Header(None), + x_trace_id: str | None = Header(None), + x_request_id: str | None = Header(None), +) -> dict: + """Recover an owned build source whose create response was lost.""" + async with _execute_endpoint( + endpoint_name="api_recover_build_sandbox", + authorization=authorization, + trace_id=x_trace_id, + request_id=x_request_id, + build_id=request.get("build_id"), + ): + from .sandbox.build_session import ModalBuildSessionService + + parsed = _parse_request(RecoverBuildSandboxRequest, request) + source_id = await ModalBuildSessionService().recover( + build_id=parsed.build_id, sandbox_backend=parsed.sandbox_backend + ) + return {"success": True, "data": {"provider_session_id": source_id}} diff --git a/packages/modal-infra/tests/test_build_sandbox_lifecycle.py b/packages/modal-infra/tests/test_build_sandbox_lifecycle.py index 47c3937515..8ea508955b 100644 --- a/packages/modal-infra/tests/test_build_sandbox_lifecycle.py +++ b/packages/modal-infra/tests/test_build_sandbox_lifecycle.py @@ -8,7 +8,10 @@ import pytest -from sandbox_runtime.constants import IMAGE_BUILD_EXECUTION_TIMEOUT_ENV_VAR +from sandbox_runtime.constants import ( + DOCKER_ENABLED_ENV_VAR, + IMAGE_BUILD_EXECUTION_TIMEOUT_ENV_VAR, +) from sandbox_runtime.modal_image_build_start import MODAL_SANDBOX_ID_ENV from sandbox_runtime.repo_image_callback import ( BUILD_ID_ENV, @@ -28,6 +31,16 @@ from src.web_api import IMAGE_BUILD_FINALIZATION_GRACE_SECONDS +@pytest.fixture(autouse=True) +def no_remote_named_lookup(monkeypatch): + from modal.exception import NotFoundError + + monkeypatch.setattr("src.sandbox.build_session.modal.Sandbox.from_name", _async_method()) + from src.sandbox.build_session import modal + + modal.Sandbox.from_name.aio.side_effect = NotFoundError("not found") + + def _async_method(return_value=None): method = MagicMock() method.aio = AsyncMock(return_value=return_value) @@ -162,7 +175,7 @@ async def test_create_build_sandbox_runs_gated_entrypoint_and_scrubs_callback_en create = _async_method(sandbox) monkeypatch.setattr("src.sandbox.build_session.modal.Sandbox.create", create) - provider_session_id = await ModalBuildSessionService().create( + launch = await ModalBuildSessionService().create( build_id="build-1", scope_kind="repo", scope_id="acme/repo", @@ -185,7 +198,8 @@ async def test_create_build_sandbox_runs_gated_entrypoint_and_scrubs_callback_en timeout_seconds=1800, ) - assert provider_session_id == "modal-session-1" + assert launch.provider_session_id == "modal-session-1" + assert launch.sandbox_backend == "modal" args = create.aio.await_args.args kwargs = create.aio.await_args.kwargs assert args == ( @@ -195,6 +209,7 @@ async def test_create_build_sandbox_runs_gated_entrypoint_and_scrubs_callback_en "--await-modal-image-build-token-stdin-v1", ) assert kwargs["tags"] == { + "openinspect_backend": "modal", "openinspect_kind": "image-build", "openinspect_build_id": "build-1", "openinspect_scope_kind": "repo", @@ -370,3 +385,78 @@ async def test_terminate_build_sandbox_treats_provider_not_found_as_success(monk ) sandbox.terminate.aio.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("docker_enabled", [False, True]) +async def test_create_build_sandbox_selects_the_variant_from_frozen_settings( + monkeypatch, docker_enabled +): + sandbox = SimpleNamespace(object_id="modal-session-1") + create = _async_method(sandbox) + monkeypatch.setattr("src.sandbox.build_session.modal.Sandbox.create", create) + default_image = object() + docker_image = object() + monkeypatch.setattr("src.sandbox.build_session.base_image", default_image) + monkeypatch.setattr("src.images.base.docker_image", docker_image) + + launch = await ModalBuildSessionService().create( + build_id="build-1", + scope_kind="repo", + scope_id="acme/repo", + repositories=[{"repo_owner": "acme", "repo_name": "repo", "branch": "main"}], + callback_url="https://cp.test/image-builds/build-complete", + failure_callback_url="https://cp.test/image-builds/build-failed", + user_env_vars={DOCKER_ENABLED_ENV_VAR: "true"}, + sandbox_backend="modal-vm" if docker_enabled else "modal", + sandbox_settings=({"cpuCores": 2, "memoryMib": 4096} if docker_enabled else None), + ) + + assert launch.provider_session_id == "modal-session-1" + assert launch.sandbox_backend == ("modal-vm" if docker_enabled else "modal") + kwargs = create.aio.await_args.kwargs + assert kwargs["env"][DOCKER_ENABLED_ENV_VAR] == ("true" if docker_enabled else "false") + if docker_enabled: + assert kwargs["image"] is docker_image + assert kwargs["experimental_options"] == {"vm_runtime": True} + assert (kwargs["cpu"], kwargs["memory"]) == (2.0, 4096) + else: + assert kwargs["image"] is default_image + assert "experimental_options" not in kwargs + + +@pytest.mark.asyncio +async def test_build_create_retry_adopts_only_owned_backend_allocation(monkeypatch): + from src.sandbox.build_session import ModalBuildSessionService + + tags = { + "openinspect_kind": "image-build", + "openinspect_build_id": "build-1", + "openinspect_scope_kind": "repo", + "openinspect_scope_id": "acme/repo", + "openinspect_launch_protocol": "stdin-token-v1", + "openinspect_backend": "modal-vm", + } + sandbox = SimpleNamespace(object_id="sb-existing", get_tags=_async_method(tags)) + lookup = _async_method(sandbox) + create = _async_method() + monkeypatch.setattr("src.sandbox.build_session.modal.Sandbox.from_name", lookup) + monkeypatch.setattr("src.sandbox.build_session.modal.Sandbox.create", create) + service = ModalBuildSessionService() + recovered = await service.recover(build_id="build-1", sandbox_backend="modal-vm") + assert recovered == "sb-existing" + launch = await service.create( + build_id="build-1", + sandbox_backend="modal-vm", + scope_kind="repo", + scope_id="acme/repo", + repositories=[{"repo_owner": "acme", "repo_name": "repo", "branch": "main"}], + callback_url="https://cp.test/complete", + failure_callback_url="https://cp.test/failed", + ) + assert launch.provider_session_id == recovered + create.aio.assert_not_awaited() + tags["openinspect_backend"] = "modal" + with pytest.raises(RuntimeError, match="ownership"): + await service.recover(build_id="build-1", sandbox_backend="modal-vm") + create.aio.assert_not_awaited() diff --git a/packages/modal-infra/tests/test_deploy.py b/packages/modal-infra/tests/test_deploy.py index 0777c9622b..e7ac2a5470 100644 --- a/packages/modal-infra/tests/test_deploy.py +++ b/packages/modal-infra/tests/test_deploy.py @@ -242,3 +242,124 @@ def test_src_modal_deploy_builds_sandbox_image_before_app_deploy(tmp_path: Path) "run python deploy.py --build-sandbox-image", "run modal deploy -m src", ] + + +def _verifying_sandbox(*, exit_codes: list[int]) -> Mock: + sandbox = Mock() + processes = [] + for code in exit_codes: + process = Mock(returncode=code) + process.stdout.read.return_value = "" + process.stderr.read.return_value = "boom" + processes.append(process) + sandbox.exec.side_effect = processes + return sandbox + + +@pytest.mark.parametrize("docker_verification_passes", [True, False]) +def test_docker_image_is_built_and_verified_on_the_vm_after_the_default_is_published( + monkeypatch, tmp_path, docker_verification_passes +) -> None: + deployed_app = object() + monkeypatch.setattr(deploy.modal.App, "lookup", Mock(return_value=deployed_app)) + monkeypatch.setattr(deploy, "base_image", Mock(build=Mock(), object_id="im-default")) + docker_build = Mock() + monkeypatch.setattr(deploy, "docker_image", Mock(build=docker_build, object_id="im-docker")) + plan = {"buildHash": "packed-recipe", "runtimeEnv": {"PACKED_PLAN": "true"}} + monkeypatch.setattr(deploy, "base_image_plan", plan) + default_sandbox = _verifying_sandbox(exit_codes=[0]) + docker_sandbox = _verifying_sandbox(exit_codes=[0, 0 if docker_verification_passes else 1]) + create = Mock(side_effect=[default_sandbox, docker_sandbox]) + monkeypatch.setattr(deploy.modal.Sandbox, "create", create) + record_path = tmp_path / "selected.json" + monkeypatch.setattr(deploy, "image_reference_path", lambda: record_path) + monkeypatch.setenv("OPENINSPECT_IMAGE_RESULT", str(tmp_path / "candidate.json")) + + if docker_verification_passes: + deploy.build_sandbox_image(with_docker=True) + else: + with pytest.raises(RuntimeError, match="verification failed"): + deploy.build_sandbox_image(with_docker=True) + + docker_build.assert_called_once_with(deployed_app) + assert "experimental_options" not in create.call_args_list[0].kwargs + vm_kwargs = create.call_args_list[1].kwargs + assert vm_kwargs["experimental_options"] == {"vm_runtime": True} + assert (vm_kwargs["cpu"], vm_kwargs["memory"]) == (2, 4096) + assert [call.args[1] for call in docker_sandbox.exec.call_args_list] == [ + "/app/verify/smoke_test.py", + "/app/verify/docker_smoke.py", + ][: len(docker_sandbox.exec.call_args_list)] + default_sandbox.terminate.assert_called_once() + docker_sandbox.terminate.assert_called_once() + record = json.loads(record_path.read_text()) + expected = {"imageId": "im-default", "buildHash": "packed-recipe"} + if docker_verification_passes: + expected["dockerImageId"] = "im-docker" + # A failed Docker variant never disturbs the verified default reference. + assert record == expected + + +def test_deployed_environment_carries_the_docker_image_when_provisioned( + monkeypatch, tmp_path +) -> None: + from src.images import base + + record_path = tmp_path / "built.json" + monkeypatch.setattr(base.modal, "is_local", lambda: True) + monkeypatch.setattr(base, "image_reference_path", lambda: record_path) + monkeypatch.setattr( + base, "local_image_plan", lambda: (tmp_path, {"buildHash": "current-recipe"}) + ) + monkeypatch.delenv("BUILD_MODAL_VM_IMAGE", raising=False) + + record_path.write_text(json.dumps({"buildHash": "current-recipe", "imageId": "im-built"})) + assert base.deployed_image_environment() == {base.IMAGE_ID_ENV: "im-built"} + + record_path.write_text( + json.dumps( + {"buildHash": "current-recipe", "imageId": "im-built", "dockerImageId": "im-docker"} + ) + ) + assert base.deployed_image_environment() == { + base.IMAGE_ID_ENV: "im-built", + base.DOCKER_IMAGE_ID_ENV: "im-docker", + } + + record_path.write_text(json.dumps({"buildHash": "current-recipe", "imageId": "im-built"})) + monkeypatch.setenv("BUILD_MODAL_VM_IMAGE", "true") + with pytest.raises(RuntimeError, match="Docker sandbox image"): + base.deployed_image_environment() + + +def test_docker_image_is_absent_in_a_deployment_that_never_provisioned_it(monkeypatch) -> None: + from src.images import base + + monkeypatch.setattr(base.modal, "is_local", lambda: False) + monkeypatch.delenv(base.DOCKER_IMAGE_ID_ENV, raising=False) + assert base._define_docker_image() is None + + monkeypatch.setenv(base.DOCKER_IMAGE_ID_ENV, "im-docker") + monkeypatch.setattr(base.modal.Image, "from_id", Mock(return_value="docker-image")) + assert base._define_docker_image() == "docker-image" + + +def test_local_docker_image_is_the_default_image_plus_the_docker_phase(monkeypatch) -> None: + from src.images import base + + image = Mock() + monkeypatch.setattr(base.modal, "is_local", lambda: True) + monkeypatch.setattr(base, "base_image", image) + + base._define_docker_image() + + image.run_commands.assert_called_once_with( + "bash /tmp/openinspect-image/packages/sandbox-images/install/install.sh docker" + ) + + +def test_docker_verification_resources_match_backend_defaults() -> None: + from src.sandbox.launch_policy import VM_DEFAULT_CPU_CORES, VM_DEFAULT_MEMORY_MIB + + assert deploy.DOCKER_VERIFICATION_CPU_CORES == VM_DEFAULT_CPU_CORES + assert deploy.DOCKER_VERIFICATION_MEMORY_MIB == VM_DEFAULT_MEMORY_MIB diff --git a/packages/modal-infra/tests/test_docker_launch.py b/packages/modal-infra/tests/test_docker_launch.py new file mode 100644 index 0000000000..44b371f2f2 --- /dev/null +++ b/packages/modal-infra/tests/test_docker_launch.py @@ -0,0 +1,113 @@ +"""Provider-local mapping from backend identity and resources to launch mechanics.""" + +import re + +import pytest + +from sandbox_runtime.constants import DOCKER_ENABLED_ENV_VAR +from src.images import base +from src.sandbox.launch_policy import ( + DockerImageUnavailableError, + InvalidDockerSettingsError, + ModalLaunch, + docker_allocation_name, + docker_allocation_tags, + docker_base_image, + docker_runtime_env, + launch_kwargs, + parse_launch, +) + + +@pytest.mark.parametrize("settings", [None, {}]) +def test_standard_defaults_preserve_existing_launch(settings): + launch = parse_launch("modal", settings) + + assert launch == ModalLaunch(backend="modal") + assert launch_kwargs(launch) == {} + assert docker_runtime_env(launch) == {DOCKER_ENABLED_ENV_VAR: "false"} + + +def test_launch_policy_maps_vm_backend_and_resources(): + launch = parse_launch("modal-vm", {"cpuCores": 2, "memoryMib": 4096}) + + assert launch == ModalLaunch(backend="modal-vm", cpu_cores=2.0, memory_mib=4096) + assert launch_kwargs(launch) == { + "experimental_options": {"vm_runtime": True}, + "cpu": 2.0, + "memory": 4096, + } + assert docker_runtime_env(launch) == {DOCKER_ENABLED_ENV_VAR: "true"} + + +@pytest.mark.parametrize( + "settings", + [ + {"dockerEnabled": "true"}, + {"dockerEnabled": 1}, + {"dockerEnabled": None}, + {"dockerEnabled": True}, + {"cpuCores": -1}, + {"cpuCores": 0, "memoryMib": 4096}, + {"cpuCores": True, "memoryMib": 4096}, + {"cpuCores": 2, "memoryMib": "4096"}, + {"cpuCores": 2, "memoryMib": 4096.5}, + {"cpuCores": float("inf"), "memoryMib": 4096}, + ], +) +def test_malformed_resources_or_removed_settings_are_rejected(settings): + with pytest.raises(InvalidDockerSettingsError): + parse_launch("modal-vm", settings) + + +def test_docker_base_image_requires_provisioning(monkeypatch): + monkeypatch.setattr(base, "docker_image", None) + with pytest.raises(DockerImageUnavailableError): + docker_base_image() + + sentinel = object() + monkeypatch.setattr(base, "docker_image", sentinel) + assert docker_base_image() is sentinel + + +def test_allocation_name_is_deterministic_per_generation_and_modal_safe(): + name = docker_allocation_name("session/with:odd chars", "sandbox-acme-repo-1700000000000") + + assert name == docker_allocation_name( + "session/with:odd chars", "sandbox-acme-repo-1700000000000" + ) + assert name != docker_allocation_name( + "session/with:odd chars", "sandbox-acme-repo-1700000000001" + ) + assert name != docker_allocation_name("other-session", "sandbox-acme-repo-1700000000000") + assert len(name) <= 64 + assert re.fullmatch(r"[a-zA-Z0-9-_.]+", name) + assert not re.fullmatch(r"ap-[a-zA-Z0-9]{22}", name) + + +def test_allocation_tags_bind_session_generation_and_backend(): + tags = docker_allocation_tags("session-1", "sandbox-1") + + assert tags["openinspect_kind"] == "session" + assert tags["openinspect_backend"] == "modal-vm" + assert tags != docker_allocation_tags("session-1", "sandbox-2") + for value in tags.values(): + assert re.fullmatch(r"[a-zA-Z0-9._-]{1,63}", value) + + +@pytest.mark.parametrize("settings", [None, {}, {"cpuCores": None, "memoryMib": None}]) +def test_vm_owns_defaults_for_absent_or_null_resources(settings): + assert launch_kwargs(parse_launch("modal-vm", settings)) == { + "cpu": 2, + "memory": 4096, + "experimental_options": {"vm_runtime": True}, + } + + +@pytest.mark.parametrize( + "settings", [{"memoryMib": True}, {"memoryMib": 0}, {"cpuCores": float("nan")}] +) +def test_both_backends_validate_explicit_resources(settings): + for backend in ("modal", "modal-vm"): + with pytest.raises(InvalidDockerSettingsError): + parse_launch(backend, settings) diff --git a/packages/modal-infra/tests/test_sandbox_launch.py b/packages/modal-infra/tests/test_sandbox_launch.py index e2f7e2d06b..02e1cd442d 100644 --- a/packages/modal-infra/tests/test_sandbox_launch.py +++ b/packages/modal-infra/tests/test_sandbox_launch.py @@ -1,5 +1,6 @@ """Behavior matrix for shared fresh, repository-image, and snapshot launches.""" +import asyncio import json from types import SimpleNamespace from unittest.mock import AsyncMock, Mock @@ -8,12 +9,19 @@ from sandbox_runtime.constants import ( CODE_SERVER_PORT_ENV_VAR, + DOCKER_ENABLED_ENV_VAR, EXPECTED_TUNNEL_PORTS_ENV_VAR, NOVNC_PORT_ENV_VAR, TTYD_PROXY_PORT_ENV_VAR, VNC_PASSWORD_ENV_VAR, ) from sandbox_runtime.types import SessionConfig +from src.sandbox.launch_policy import ( + DockerImageUnavailableError, + InvalidDockerSettingsError, + docker_allocation_name, + docker_allocation_tags, +) from src.sandbox.manager import ( RepositoryImageUnavailableError, SandboxConfig, @@ -136,6 +144,11 @@ async def test_launch_matrix_preserves_common_and_source_specific_behavior( assert kwargs["cpu"] == 1.5 assert kwargs["memory"] == 3072 assert kwargs["encrypted_ports"] == [9000, 9001, 9002, 3000] + # The default launch never touches the VM runtime or named allocations. + assert "experimental_options" not in kwargs + assert "name" not in kwargs + assert "tags" not in kwargs + assert env[DOCKER_ENABLED_ENV_VAR] == "false" assert env["CONTROL_PLANE_URL"] == "https://control.example" assert env["CUSTOM_ENV"] == "preserved" @@ -218,3 +231,328 @@ async def test_repository_image_not_found_is_reported_explicitly(monkeypatch): repo_image_id="repo-image-missing", ) ) + + +DOCKER_SETTINGS = {"cpuCores": 2, "memoryMib": 4096} + + +def _docker_manager(monkeypatch) -> tuple[SandboxManager, dict, object]: + captured: dict = {} + docker_image = object() + monkeypatch.setattr("src.sandbox.manager.base_image", object()) + monkeypatch.setattr("src.images.base.docker_image", docker_image) + monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", _fake_create(captured)) + monkeypatch.setattr( + SandboxManager, + "_resolve_and_setup_tunnels", + AsyncMock(return_value=(None, None, None, {})), + ) + return SandboxManager(), captured, docker_image + + +def _docker_config(**overrides) -> SandboxConfig: + fields = { + "repo_owner": "acme", + "repo_name": "repo", + "sandbox_id": "sandbox-acme-repo-1700000000000", + "session_config": SessionConfig( + session_id="session-1", repo_owner="acme", repo_name="repo" + ), + "control_plane_url": "https://control.example", + "sandbox_auth_token": "token", + "user_env_vars": {DOCKER_ENABLED_ENV_VAR: "false", "CUSTOM_ENV": "preserved"}, + "settings": dict(DOCKER_SETTINGS), + "sandbox_backend": "modal-vm", + } + return SandboxConfig(**{**fields, **overrides}) + + +def _not_found(*_args, **_kwargs): + from modal.exception import NotFoundError + + raise NotFoundError("no sandbox") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("image_source", ["base", "repository", "snapshot"]) +async def test_docker_launch_selects_vm_runtime_and_named_allocation(monkeypatch, image_source): + manager, captured, docker_image = _docker_manager(monkeypatch) + artifact = object() + monkeypatch.setattr("src.sandbox.manager.modal.Image.from_id", lambda _id: artifact) + monkeypatch.setattr( + "src.sandbox.manager.modal.Sandbox.from_name", + SimpleNamespace(aio=AsyncMock(side_effect=_not_found)), + ) + + if image_source == "snapshot": + handle = await manager.restore_from_snapshot( + snapshot_image_id="snapshot-1", + session_config={"session_id": "session-1", "repo_owner": "acme", "repo_name": "repo"}, + sandbox_id="sandbox-acme-repo-1700000000000", + control_plane_url="https://control.example", + sandbox_auth_token="token", + user_env_vars={DOCKER_ENABLED_ENV_VAR: "false"}, + settings=dict(DOCKER_SETTINGS), + sandbox_backend="modal-vm", + ) + else: + handle = await manager.create_sandbox( + _docker_config(repo_image_id="repo-image-1" if image_source == "repository" else None) + ) + + kwargs = captured["kwargs"] + assert kwargs["image"] is (docker_image if image_source == "base" else artifact) + assert kwargs["experimental_options"] == {"vm_runtime": True} + assert kwargs["cpu"] == 2.0 + assert kwargs["memory"] == 4096 + assert kwargs["name"] == docker_allocation_name("session-1", "sandbox-acme-repo-1700000000000") + assert kwargs["tags"] == docker_allocation_tags("session-1", "sandbox-acme-repo-1700000000000") + # The trusted signal wins over any user-supplied value. + assert kwargs["env"][DOCKER_ENABLED_ENV_VAR] == "true" + assert handle.sandbox_backend == "modal-vm" + + +@pytest.mark.asyncio +async def test_docker_launch_without_a_provisioned_image_never_uses_the_default(monkeypatch): + manager, captured, _ = _docker_manager(monkeypatch) + monkeypatch.setattr("src.images.base.docker_image", None) + + with pytest.raises(DockerImageUnavailableError): + await manager.create_sandbox(_docker_config()) + + assert "kwargs" not in captured + + +@pytest.mark.asyncio +async def test_malformed_docker_setting_fails_before_any_launch(monkeypatch): + manager, captured, _ = _docker_manager(monkeypatch) + + with pytest.raises(InvalidDockerSettingsError): + await manager.create_sandbox(_docker_config(settings={"dockerEnabled": "true"})) + + assert "kwargs" not in captured + + +@pytest.mark.asyncio +async def test_docker_launch_adopts_an_existing_owned_allocation(monkeypatch): + manager, captured, _ = _docker_manager(monkeypatch) + tags = docker_allocation_tags("session-1", "sandbox-acme-repo-1700000000000") + existing = SimpleNamespace(object_id="modal-existing", get_tags=AsyncMock(return_value=tags)) + existing.get_tags.aio = existing.get_tags + from_name = AsyncMock(return_value=existing) + monkeypatch.setattr( + "src.sandbox.manager.modal.Sandbox.from_name", SimpleNamespace(aio=from_name) + ) + + handle = await manager.create_sandbox(_docker_config()) + + assert "kwargs" not in captured + assert handle.modal_object_id == "modal-existing" + from_name.assert_awaited_once_with( + "open-inspect", docker_allocation_name("session-1", "sandbox-acme-repo-1700000000000") + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("create_race", [False, True]) +@pytest.mark.parametrize("image_source", ["base", "snapshot"]) +async def test_docker_retry_returns_the_original_access_credentials( + monkeypatch, create_race, image_source +): + from modal.exception import AlreadyExistsError, NotFoundError + + manager, captured, _ = _docker_manager(monkeypatch) + monkeypatch.setattr("src.sandbox.manager.modal.Image.from_id", lambda _id: object()) + monkeypatch.setattr( + SandboxManager, "_generate_code_server_password", Mock(side_effect=["original", "new"]) + ) + monkeypatch.setattr( + SandboxManager, "_generate_vnc_password", Mock(side_effect=["old-vnc", "new-vnc"]) + ) + from_name = AsyncMock(side_effect=NotFoundError("not created")) + monkeypatch.setattr( + "src.sandbox.manager.modal.Sandbox.from_name", SimpleNamespace(aio=from_name) + ) + + async def launch(): + config = _docker_config(code_server_enabled=True, vnc_enabled=True) + if image_source == "base": + return await manager.create_sandbox(config) + return await manager.restore_from_snapshot( + snapshot_image_id="snapshot-1", + session_config=config.session_config, + sandbox_id=config.sandbox_id, + code_server_enabled=True, + vnc_enabled=True, + settings=config.settings, + sandbox_backend=config.sandbox_backend, + ) + + original = await launch() + original_env = captured["kwargs"]["env"] + credential_output = json.dumps( + {key: original_env[key] for key in ("CODE_SERVER_PASSWORD", VNC_PASSWORD_ENV_VAR)} + ) + process = SimpleNamespace( + stdout=SimpleNamespace(read=SimpleNamespace(aio=AsyncMock(return_value=credential_output))), + wait=SimpleNamespace(aio=AsyncMock(return_value=0)), + ) + existing = SimpleNamespace( + object_id=original.modal_object_id, + get_tags=SimpleNamespace(aio=AsyncMock(return_value=captured["kwargs"]["tags"])), + exec=SimpleNamespace(aio=AsyncMock(return_value=process)), + ) + from_name.side_effect = [NotFoundError("racing"), existing] if create_race else [existing] + create = AsyncMock(side_effect=AlreadyExistsError("already created")) + monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.create", SimpleNamespace(aio=create)) + + adopted = await launch() + + assert adopted.modal_object_id == original.modal_object_id + assert adopted.code_server_password == original.code_server_password == "original" + assert adopted.vnc_password == original.vnc_password == "old-vnc" + assert create.await_count == int(create_race) + assert existing.exec.aio.call_args.args[-2:] == ("CODE_SERVER_PASSWORD", VNC_PASSWORD_ENV_VAR) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "output,exit_code", [("{}", 0), ('{"CODE_SERVER_PASSWORD": ""}', 0), ("invalid", 0), ("", 1)] +) +async def test_docker_adoption_fails_if_original_credentials_cannot_be_recovered( + monkeypatch, output, exit_code +): + manager, captured, _ = _docker_manager(monkeypatch) + process = SimpleNamespace( + stdout=SimpleNamespace(read=SimpleNamespace(aio=AsyncMock(return_value=output))), + wait=SimpleNamespace(aio=AsyncMock(return_value=exit_code)), + ) + existing = SimpleNamespace( + object_id="modal-existing", + get_tags=SimpleNamespace( + aio=AsyncMock( + return_value=docker_allocation_tags("session-1", "sandbox-acme-repo-1700000000000") + ) + ), + exec=SimpleNamespace(aio=AsyncMock(return_value=process)), + ) + monkeypatch.setattr( + "src.sandbox.manager.modal.Sandbox.from_name", + SimpleNamespace(aio=AsyncMock(return_value=existing)), + ) + + with pytest.raises(RuntimeError, match="Could not recover adopted sandbox access credentials"): + await manager.create_sandbox(_docker_config(code_server_enabled=True)) + + assert "kwargs" not in captured + manager._resolve_and_setup_tunnels.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_docker_launch_refuses_a_same_named_allocation_it_does_not_own(monkeypatch): + manager, captured, _ = _docker_manager(monkeypatch) + foreign = SimpleNamespace( + object_id="modal-foreign", + get_tags=AsyncMock(return_value={"openinspect_kind": "session"}), + ) + foreign.get_tags.aio = foreign.get_tags + monkeypatch.setattr( + "src.sandbox.manager.modal.Sandbox.from_name", + SimpleNamespace(aio=AsyncMock(return_value=foreign)), + ) + + with pytest.raises(RuntimeError, match="ownership mismatch"): + await manager.create_sandbox(_docker_config()) + + assert "kwargs" not in captured + + +@pytest.mark.asyncio +async def test_docker_launch_retires_the_prior_generation_only_when_owned(monkeypatch): + manager, captured, _ = _docker_manager(monkeypatch) + prior_tags = docker_allocation_tags("session-1", "sandbox-acme-repo-1699999999999") + prior = SimpleNamespace( + object_id="modal-prior", + get_tags=AsyncMock(return_value=prior_tags), + terminate=AsyncMock(), + ) + prior.get_tags.aio = prior.get_tags + prior.terminate.aio = prior.terminate + prior_name = docker_allocation_name("session-1", "sandbox-acme-repo-1699999999999") + + async def from_name(_app, name): + if name == prior_name: + return prior + _not_found() + + monkeypatch.setattr( + "src.sandbox.manager.modal.Sandbox.from_name", SimpleNamespace(aio=from_name) + ) + + await manager.create_sandbox( + _docker_config(retire_sandbox_id="sandbox-acme-repo-1699999999999") + ) + + prior.terminate.assert_awaited_once_with(wait=True) + assert captured["kwargs"]["name"] == docker_allocation_name( + "session-1", "sandbox-acme-repo-1700000000000" + ) + + # A prior allocation with foreign tags is left alone. + prior.terminate.reset_mock() + prior.get_tags = AsyncMock(return_value={"openinspect_kind": "other"}) + prior.get_tags.aio = prior.get_tags + await manager.create_sandbox( + _docker_config(retire_sandbox_id="sandbox-acme-repo-1699999999999") + ) + prior.terminate.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("termination_fails", [False, True]) +async def test_docker_successor_waits_for_confirmed_predecessor_retirement( + monkeypatch, termination_fails +): + manager, captured, _ = _docker_manager(monkeypatch) + termination_requested = asyncio.Event() + termination_finished = asyncio.Event() + + async def terminate(*, wait=False): + termination_requested.set() + if wait: + await termination_finished.wait() + if termination_fails: + raise RuntimeError("termination unconfirmed") + + prior = SimpleNamespace( + object_id="modal-prior", + get_tags=SimpleNamespace( + aio=AsyncMock(return_value=docker_allocation_tags("session-1", "sandbox-prior")) + ), + terminate=SimpleNamespace(aio=terminate), + ) + from modal.exception import NotFoundError + + monkeypatch.setattr( + "src.sandbox.manager.modal.Sandbox.from_name", + SimpleNamespace(aio=AsyncMock(side_effect=[prior, NotFoundError("no successor")])), + ) + launch = asyncio.create_task( + manager.create_sandbox(_docker_config(retire_sandbox_id="sandbox-prior")) + ) + try: + await asyncio.wait_for(termination_requested.wait(), timeout=1) + assert not launch.done() + assert "kwargs" not in captured + + termination_finished.set() + if termination_fails: + with pytest.raises(RuntimeError, match="termination unconfirmed"): + await launch + assert "kwargs" not in captured + else: + await launch + assert "kwargs" in captured + finally: + launch.cancel() + await asyncio.gather(launch, return_exceptions=True) diff --git a/packages/modal-infra/tests/test_sandbox_resources.py b/packages/modal-infra/tests/test_sandbox_resources.py index cc2aa8b105..cb8d7a1b90 100644 --- a/packages/modal-infra/tests/test_sandbox_resources.py +++ b/packages/modal-infra/tests/test_sandbox_resources.py @@ -4,26 +4,27 @@ import pytest -from src.sandbox.manager import SandboxConfig, SandboxManager, _resource_kwargs +from src.sandbox.launch_policy import launch_kwargs, parse_launch +from src.sandbox.manager import SandboxConfig, SandboxManager class TestResourceKwargs: - """_resource_kwargs maps sandbox settings to Modal create kwargs.""" + """Provider-local launch policy maps resource settings to Modal kwargs.""" def test_empty_settings(self): - assert _resource_kwargs({}) == {} + assert launch_kwargs(parse_launch("modal", {})) == {} def test_maps_cpu_and_memory(self): - assert _resource_kwargs({"cpuCores": 2, "memoryMib": 4096}) == { + assert launch_kwargs(parse_launch("modal", {"cpuCores": 2, "memoryMib": 4096})) == { "cpu": 2.0, "memory": 4096, } def test_allows_fractional_cpu(self): - assert _resource_kwargs({"cpuCores": 0.5}) == {"cpu": 0.5} + assert launch_kwargs(parse_launch("modal", {"cpuCores": 0.5})) == {"cpu": 0.5} def test_independent_fields(self): - assert _resource_kwargs({"memoryMib": 2048}) == {"memory": 2048} + assert launch_kwargs(parse_launch("modal", {"memoryMib": 2048})) == {"memory": 2048} def _fake_create(captured: dict): diff --git a/packages/modal-infra/tests/test_snapshot_timeout.py b/packages/modal-infra/tests/test_snapshot_timeout.py index ef931d3167..cdd2a41f7e 100644 --- a/packages/modal-infra/tests/test_snapshot_timeout.py +++ b/packages/modal-infra/tests/test_snapshot_timeout.py @@ -42,7 +42,7 @@ async def test_take_snapshot_passes_explicit_timeout(): @pytest.mark.asyncio async def test_get_sandbox_by_id_awaits_async_lookup(monkeypatch): - modal_sandbox = SimpleNamespace() + modal_sandbox = SimpleNamespace(get_tags=_async_method({})) from_id = _async_method(modal_sandbox) monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.from_id", from_id) @@ -142,3 +142,62 @@ async def test_stop_sandbox_propagates_explicit_cancellation(monkeypatch): with pytest.raises(asyncio.CancelledError): await SandboxManager().stop_sandbox("sandbox-1") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("exit_code", [0, 1]) +async def test_vm_capture_requires_docker_preparation(exit_code): + process = SimpleNamespace(wait=_async_method(exit_code)) + execute = _async_method(process) + snapshot = _async_method(SimpleNamespace(object_id="im-vm")) + handle = SandboxHandle( + sandbox_id="sb-vm", + sandbox_backend="modal-vm", + status=SandboxStatus.READY, + created_at=0, + modal_sandbox=SimpleNamespace(exec=execute, snapshot_filesystem=snapshot), + ) + if exit_code: + with pytest.raises(RuntimeError, match="preparation"): + await SandboxManager().take_snapshot(handle) + snapshot.aio.assert_not_awaited() + else: + assert await SandboxManager().take_snapshot(handle) == "im-vm" + snapshot.aio.assert_awaited_once() + assert execute.aio.call_args.args == ( + "python", + "-m", + "sandbox_runtime.docker_control", + "prepare", + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("elapsed", [9, 10]) +async def test_vm_preparation_consumes_capture_budget(monkeypatch, elapsed): + clock = SimpleNamespace(time=lambda: 1000, monotonic=lambda: 0) + monkeypatch.setattr("src.sandbox.manager.time", clock) + + async def prepare(): + clock.monotonic = lambda: elapsed + return 0 + + wait = _async_method() + wait.aio.side_effect = prepare + snapshot = _async_method(SimpleNamespace(object_id="im-vm")) + handle = SandboxHandle( + sandbox_id="sb-vm", + sandbox_backend="modal-vm", + status=SandboxStatus.READY, + created_at=0, + modal_sandbox=SimpleNamespace( + exec=_async_method(SimpleNamespace(wait=wait)), snapshot_filesystem=snapshot + ), + ) + if elapsed == 10: + with pytest.raises(TimeoutError): + await SandboxManager().take_snapshot(handle, timeout_seconds=10) + snapshot.aio.assert_not_awaited() + else: + await SandboxManager().take_snapshot(handle, timeout_seconds=10) + snapshot.aio.assert_awaited_once_with(timeout=1) diff --git a/packages/modal-infra/tests/test_web_api_build_sandbox.py b/packages/modal-infra/tests/test_web_api_build_sandbox.py index 4ca2d645d0..728c67e07c 100644 --- a/packages/modal-infra/tests/test_web_api_build_sandbox.py +++ b/packages/modal-infra/tests/test_web_api_build_sandbox.py @@ -9,7 +9,11 @@ from sandbox_runtime.types import SandboxStatus from src import web_api -from src.sandbox.build_session import DEFAULT_BUILD_TIMEOUT_SECONDS, MAX_BUILD_TIMEOUT_SECONDS +from src.sandbox.build_session import ( + DEFAULT_BUILD_TIMEOUT_SECONDS, + MAX_BUILD_TIMEOUT_SECONDS, + BuildSessionLaunch, +) from src.sandbox.manager import SandboxHandle, SandboxManager REPOSITORIES = [{"repo_owner": "acme", "repo_name": "repo", "branch": "main"}] @@ -23,7 +27,7 @@ def _patch_dependencies(monkeypatch: pytest.MonkeyPatch): monkeypatch.setattr(web_api, "require_auth", lambda _authorization: None) monkeypatch.setattr(web_api, "validate_control_plane_url", lambda _url: True) service = SimpleNamespace( - create=AsyncMock(return_value="modal-session-1"), + create=AsyncMock(return_value=BuildSessionLaunch("modal-session-1", "modal")), start=AsyncMock(), terminate=AsyncMock(), snapshot=AsyncMock(return_value="modal-image-1"), @@ -104,7 +108,7 @@ async def test_create_build_sandbox_forwards_callback_context_and_returns_provid assert result == { "success": True, - "data": {"provider_session_id": "modal-session-1"}, + "data": {"provider_session_id": "modal-session-1", "sandbox_backend": "modal"}, } service.create.assert_awaited_once_with( build_id="imgb-1", @@ -119,6 +123,8 @@ async def test_create_build_sandbox_forwards_callback_context_and_returns_provid user_env_vars={"FOO": "bar"}, build_execution_timeout_seconds=DEFAULT_BUILD_TIMEOUT_SECONDS, timeout_seconds=2400, + sandbox_settings=None, + sandbox_backend="modal", ) @@ -145,6 +151,36 @@ async def test_create_build_sandbox_adds_finalization_grace_to_default_timeout(m ) +@pytest.mark.asyncio +@pytest.mark.parametrize("docker_enabled", [False, True]) +async def test_create_build_sandbox_returns_the_provider_confirmed_variant( + monkeypatch, docker_enabled +): + service = _patch_dependencies(monkeypatch) + service.create.return_value = BuildSessionLaunch( + "modal-session-1", "modal-vm" if docker_enabled else "modal" + ) + settings = {"cpuCores": 2, "memoryMib": 4096} + + result = await _call( + web_api.api_create_build_sandbox, + { + "scope_kind": "repo", + "scope_id": "acme/repo", + "build_id": "imgb-1", + "repositories": REPOSITORIES, + "sandbox_settings": settings, + **CALLBACK_CONTEXT, + }, + ) + + assert result["data"] == { + "provider_session_id": "modal-session-1", + "sandbox_backend": "modal-vm" if docker_enabled else "modal", + } + assert service.create.await_args.kwargs["sandbox_settings"] == settings + + @pytest.mark.asyncio @pytest.mark.parametrize( ("payload_callbacks", "missing_field"), @@ -597,7 +633,7 @@ def reject_auth(_authorization): @pytest.mark.asyncio async def test_generic_snapshot_reason_cannot_select_build_identity_rules(monkeypatch): monkeypatch.setattr(web_api, "require_auth", lambda _authorization: None) - handle = SimpleNamespace() + handle = SimpleNamespace(sandbox_backend="modal") manager = SimpleNamespace( get_sandbox_by_id=AsyncMock(return_value=handle), take_snapshot=AsyncMock(return_value="im-session-1"), @@ -706,3 +742,29 @@ async def test_generic_snapshot_passes_ordinary_deadline_to_real_manager( assert result["data"]["image_id"] == "im-session-1" get_sandbox_by_id.assert_awaited_once_with("modal-session-1") snapshot_filesystem.aio.assert_awaited_once_with(timeout=10) + + +@pytest.mark.asyncio +async def test_vm_snapshot_retirement_shares_the_request_deadline(monkeypatch): + import asyncio + + monkeypatch.setattr(web_api, "require_auth", lambda _authorization: None) + monkeypatch.setattr(web_api, "time", SimpleNamespace(time=lambda: 1000)) + + async def delayed_termination(**kwargs): + assert kwargs == {"wait": True} + await asyncio.sleep(10) + + terminate = SimpleNamespace(aio=AsyncMock(side_effect=delayed_termination)) + handle = SimpleNamespace( + sandbox_backend="modal-vm", modal_sandbox=SimpleNamespace(terminate=terminate) + ) + manager = SimpleNamespace( + get_sandbox_by_id=AsyncMock(return_value=handle), + take_snapshot=AsyncMock(return_value="im-vm"), + ) + monkeypatch.setattr("src.sandbox.manager.SandboxManager", lambda: manager) + with pytest.raises(web_api.HTTPException) as exc: + await _call_generic_snapshot({"sandbox_id": "sb-vm", "deadline_at_ms": 1_000_020}) + assert exc.value.status_code == 408 + terminate.aio.assert_awaited_once_with(wait=True) diff --git a/packages/modal-infra/tests/test_web_api_create_sandbox.py b/packages/modal-infra/tests/test_web_api_create_sandbox.py index 86b51a95e0..ffcc4a5ad6 100644 --- a/packages/modal-infra/tests/test_web_api_create_sandbox.py +++ b/packages/modal-infra/tests/test_web_api_create_sandbox.py @@ -10,6 +10,7 @@ from sandbox_runtime.types import SandboxStatus from src import web_api from src.sandbox import manager as manager_module +from src.sandbox.launch_policy import DockerImageUnavailableError, InvalidDockerSettingsError from src.sandbox.manager import DEFAULT_SANDBOX_TIMEOUT_SECONDS, RepositoryImageUnavailableError @@ -39,6 +40,7 @@ async def create_sandbox(self, config): vnc_password=vnc_password, ttyd_url=None, tunnel_urls=None, + sandbox_backend="modal", ) monkeypatch.setattr(manager_module, "SandboxManager", FakeManager) @@ -64,6 +66,7 @@ async def restore_from_snapshot(self, **kwargs): vnc_password=vnc_password, ttyd_url=None, tunnel_urls=None, + sandbox_backend="modal", ) monkeypatch.setattr(manager_module, "SandboxManager", FakeManager) @@ -796,3 +799,72 @@ def test_session_config_helper_ignores_null_wire_values(): assert config.provider == "anthropic" assert config.model == "claude-sonnet-4-6" assert config.branch is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("call", "payload", "captured_key", "config_reader"), + [ + ( + _call_create_sandbox, + CREATE_REQUEST, + "config", + lambda captured: (captured["config"].settings, captured["config"].retire_sandbox_id), + ), + ( + _call_restore_sandbox, + RESTORE_REQUEST, + "restore", + lambda captured: ( + captured["restore"]["settings"], + captured["restore"]["retire_sandbox_id"], + ), + ), + ], +) +async def test_sandbox_requests_forward_docker_settings_and_report_the_launch( + monkeypatch, call, payload, captured_key, config_reader +): + _patch_auth(monkeypatch) + captured: dict = {} + if captured_key == "config": + _patch_manager(monkeypatch, captured) + else: + _patch_restore_manager(monkeypatch, captured) + settings = {"cpuCores": 2, "memoryMib": 4096} + + result = await call( + {**payload, "sandbox_settings": settings, "retire_sandbox_id": "sandbox-prior"} + ) + + assert config_reader(captured) == (settings, "sandbox-prior") + # The fake handle launched standard; the response reports what actually happened. + assert result["data"]["sandbox_backend"] == "modal" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("error", "status", "detail"), + [ + ( + InvalidDockerSettingsError("dockerEnabled must be a boolean"), + 400, + "dockerEnabled must be a boolean", + ), + (DockerImageUnavailableError("not provisioned"), 501, "docker_not_available"), + ], +) +async def test_docker_launch_errors_map_to_actionable_statuses(monkeypatch, error, status, detail): + _patch_auth(monkeypatch) + + class FailingManager: + async def create_sandbox(self, config): + raise error + + monkeypatch.setattr(manager_module, "SandboxManager", FailingManager) + + with pytest.raises(HTTPException) as exc_info: + await _call_create_sandbox(CREATE_REQUEST) + + assert exc_info.value.status_code == status + assert exc_info.value.detail == detail diff --git a/packages/sandbox-images/install/docker-daemon.json b/packages/sandbox-images/install/docker-daemon.json new file mode 100644 index 0000000000..3ebd8cdd1c --- /dev/null +++ b/packages/sandbox-images/install/docker-daemon.json @@ -0,0 +1,19 @@ +{ + "storage-driver": "overlay2", + "features": { + "containerd-snapshotter": false + }, + "bip": "10.200.0.1/24", + "default-address-pools": [ + { + "base": "10.201.0.0/16", + "size": 24 + } + ], + "shutdown-timeout": 20, + "log-driver": "local", + "log-opts": { + "max-size": "10m", + "max-file": "3" + } +} diff --git a/packages/sandbox-images/install/docker.sh b/packages/sandbox-images/install/docker.sh new file mode 100644 index 0000000000..74657604ea --- /dev/null +++ b/packages/sandbox-images/install/docker.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Optional Docker variant: adds a pinned Docker Engine, CLI, containerd, Buildx +# and Compose on top of the default image. Runs only while constructing the +# Docker image candidate (install.sh docker), never at session boot. +set -euo pipefail +source "$OI_INSTALL_DIR/common.sh" +if [[ "$OI_OS" != debian ]]; then + echo "The Docker variant currently requires Debian bookworm (got $OI_OS)" >&2 + exit 1 +fi +export DEBIAN_FRONTEND=noninteractive +package_dir="$(mktemp -d)" +rootfs="$(mktemp -d)" +trap 'rm -rf "$package_dir" "$rootfs"' EXIT +for tool in ENGINE CLI CONTAINERD BUILDX COMPOSE; do + file_key="DOCKER_${tool}_FILE" + hash_key="DOCKER_${tool}_SHA256" + download_checked \ + "https://download.docker.com/linux/debian/dists/bookworm/pool/stable/amd64/${!file_key}" \ + "${!hash_key}" \ + "$package_dir/${!file_key}" +done +apt-get update +apt-get install -y --no-install-recommends busybox-static "$package_dir"/*.deb +install -d -m 0755 /etc/docker +install -m 0644 "$OI_INSTALL_DIR/docker-daemon.json" /etc/docker/daemon.json +install -m 0644 "$OI_BUNDLE/packages/sandbox-images/verify/docker_smoke.py" /app/verify/docker_smoke.py +# A registry-independent BusyBox root filesystem lets image verification run +# a real build, container and Compose network without pulling anything. +mkdir -p "$rootfs/bin" +cp /bin/busybox "$rootfs/bin/busybox" +for applet in sh cat mkdir httpd wget sleep grep; do ln -s busybox "$rootfs/bin/$applet"; done +install -d -m 0755 /opt/openinspect/docker-smoke +tar --format=ustar --owner=0 --group=0 --numeric-owner -C "$rootfs" -cf /opt/openinspect/docker-smoke/rootfs.tar . +docker --version +docker buildx version +docker compose version diff --git a/packages/sandbox-images/src/sandbox_images/bundle.py b/packages/sandbox-images/src/sandbox_images/bundle.py index d42a1c7bfa..aad948f5d7 100644 --- a/packages/sandbox-images/src/sandbox_images/bundle.py +++ b/packages/sandbox-images/src/sandbox_images/bundle.py @@ -15,6 +15,7 @@ from .configuration import IMAGE_PACKAGE, RUNTIME_PACKAGE, read_json, runtime_environment from .locks import update_locks +DOCKER_PACKAGES = ("engine", "cli", "containerd", "buildx", "compose") PROVIDERS = ("modal", "daytona", "e2b", "vercel", "opencomputer") EXCLUDED = { ".terraform", @@ -99,6 +100,18 @@ def version(value: str) -> tuple[int, ...]: version(pin["version"]) if not re.fullmatch(r"[a-f0-9]{64}", pin["sha256"]): raise ValueError("Downloaded image tools must have a SHA-256 pin") + # The optional Docker variant installs pinned Debian packages, never a + # convenience script or an unpinned apt repository. + docker = tools.get("docker") + if not isinstance(docker, dict) or set(docker) != set(DOCKER_PACKAGES): + raise ValueError( + "Docker packages must pin exactly the engine, cli, containerd, buildx and compose" + ) + for pin in docker.values(): + if not re.fullmatch(r"[a-z0-9_.~+-]+_amd64\.deb", pin.get("file", "")): + raise ValueError("Docker packages must be pinned Debian amd64 artifacts") + if not re.fullmatch(r"[a-f0-9]{64}", pin.get("sha256", "")): + raise ValueError("Docker packages must have a SHA-256 pin") def source_files(root: Path, paths: tuple[Path, ...]) -> list[Path]: @@ -210,6 +223,10 @@ def pack_bundle(root: Path, provider: str, output_root: Path) -> PackedBundle: pin = toolchain[key][plan["target"]["node"]] if key == "node" else toolchain[key] variables[f"{name}_VERSION"] = pin["version"] variables[f"{name}_SHA256"] = pin["sha256"] + for package in DOCKER_PACKAGES: + pin = toolchain["docker"][package] + variables[f"DOCKER_{package.upper()}_FILE"] = pin["file"] + variables[f"DOCKER_{package.upper()}_SHA256"] = pin["sha256"] (destination / "image-config.sh").write_text( "\n".join(f"export {key}={shlex.quote(value)}" for key, value in variables.items()) + "\n" diff --git a/packages/sandbox-images/tests/test_bundle.py b/packages/sandbox-images/tests/test_bundle.py index 7ee324c120..182277c751 100644 --- a/packages/sandbox-images/tests/test_bundle.py +++ b/packages/sandbox-images/tests/test_bundle.py @@ -1,6 +1,7 @@ """Shared payload staging and conservative build invalidation.""" import json +import re import shutil from pathlib import Path @@ -146,3 +147,63 @@ def test_symlinks_and_executable_modes(checkout, tmp_path): (directory / "escape.sh").symlink_to(tmp_path / "outside") with pytest.raises(ValueError, match="symlink"): plan_image(checkout, "e2b") + + +def test_docker_variant_pins_debian_packages_by_checksum(): + tools = json.loads((REPO_ROOT / "packages/sandbox-images/toolchain.json").read_text()) + assert set(tools["docker"]) == {"engine", "cli", "containerd", "buildx", "compose"} + tools["docker"]["engine"]["file"] = "docker-ce_latest.tgz" + with pytest.raises(ValueError, match="Debian amd64"): + validate_toolchain(tools) + tools = json.loads((REPO_ROOT / "packages/sandbox-images/toolchain.json").read_text()) + tools["docker"]["compose"]["sha256"] = "deadbeef" + with pytest.raises(ValueError, match="SHA-256"): + validate_toolchain(tools) + del tools["docker"]["buildx"] + with pytest.raises(ValueError, match="exactly"): + validate_toolchain(tools) + + +def test_docker_variant_installs_only_checked_packages_and_never_runs_at_boot(): + script = (REPO_ROOT / "packages/sandbox-images/install/docker.sh").read_text() + assert "download_checked" in script + assert "get.docker.com" not in script + assert "docker pull" not in script + install_commands = [line for line in script.splitlines() if "apt-get install" in line] + assert install_commands + assert all("--no-install-recommends" in command for command in install_commands) + assert "/etc/docker/daemon.json" in script + # The default image's installer never includes the Docker phase. + install = (REPO_ROOT / "packages/sandbox-images/install/install.sh").read_text() + assert "docker" not in install + + +def test_docker_daemon_config_uses_overlay2_and_avoids_modal_networks(): + config = json.loads( + (REPO_ROOT / "packages/sandbox-images/install/docker-daemon.json").read_text() + ) + assert config["storage-driver"] == "overlay2" + assert config["features"]["containerd-snapshotter"] is False + assert config["bip"].startswith("10.") + assert all(pool["base"].startswith("10.") for pool in config["default-address-pools"]) + + +def test_docker_pins_reach_the_installer_as_shell_variables(checkout, tmp_path): + bundle = pack_bundle(checkout, "modal", tmp_path / "out") + config = (bundle.directory / "image-config.sh").read_text() + for package in ("ENGINE", "CLI", "CONTAINERD", "BUILDX", "COMPOSE"): + assert f"export DOCKER_{package}_FILE=" in config + assert f"export DOCKER_{package}_SHA256=" in config + + +def test_docker_daemon_shutdown_timeout_fits_inside_the_runtime_stop_deadline(): + config = json.loads( + (REPO_ROOT / "packages/sandbox-images/install/docker-daemon.json").read_text() + ) + runtime = ( + REPO_ROOT / "packages/sandbox-runtime/src/sandbox_runtime/docker_service.py" + ).read_text() + stop_deadline = float( + re.search(r"^DOCKER_STOP_TIMEOUT_SECONDS = ([0-9.]+)", runtime, re.M).group(1) + ) + assert config["shutdown-timeout"] < stop_deadline diff --git a/packages/sandbox-images/toolchain.json b/packages/sandbox-images/toolchain.json index c40b38c6bf..14e01c89dd 100644 --- a/packages/sandbox-images/toolchain.json +++ b/packages/sandbox-images/toolchain.json @@ -1,6 +1,28 @@ { "schemaVersion": 1, "osRefresh": "2026-09-06", + "docker": { + "engine": { + "file": "docker-ce_29.8.1-1~debian.12~bookworm_amd64.deb", + "sha256": "3e38704938b78358563164cf419d00598e32a2119efc7cea80bdec2249efaf63" + }, + "cli": { + "file": "docker-ce-cli_29.8.1-1~debian.12~bookworm_amd64.deb", + "sha256": "ff812c5853c52ef120ec73132320805d179a376e42785085e2053ce7f2479860" + }, + "containerd": { + "file": "containerd.io_2.3.5-1~debian.12~bookworm_amd64.deb", + "sha256": "f821adc67b4bd1bc8da06288ec3aa81e029cbb264bd6db569c7cdab7f8bb0701" + }, + "buildx": { + "file": "docker-buildx-plugin_0.37.1-1~debian.12~bookworm_amd64.deb", + "sha256": "cf36da1a4c31287fef2c0cb5413aa660f56d4ec183b4d24dc47cd83ebef0a2c6" + }, + "compose": { + "file": "docker-compose-plugin_5.5.1-1~debian.12~bookworm_amd64.deb", + "sha256": "0455e6a54bcc07b1bcb5d92d9e6e77dedc485433c87c1b75572e6d837a96e459" + } + }, "chrome": { "version": "152.0.7977.82", "sha256": "0704631fb3e4f741092e08f55272f90abc3e307f991f05f332924364415b02e0" diff --git a/packages/sandbox-images/verify/docker_smoke.py b/packages/sandbox-images/verify/docker_smoke.py new file mode 100644 index 0000000000..177c89bd66 --- /dev/null +++ b/packages/sandbox-images/verify/docker_smoke.py @@ -0,0 +1,153 @@ +"""Verify the Docker image variant on the runtime it will be launched with. + +Runs inside the candidate Docker image on a Modal VM, using the production +DockerService as the daemon owner. It proves the daemon reaches readiness on +overlay2, that a build, a container and a Compose network work without any +registry, and that the daemon stops cleanly the way an image build requires. +""" + +from __future__ import annotations + +import asyncio +import os +import sys +import tempfile +from pathlib import Path + +from sandbox_runtime.docker_service import DockerService +from sandbox_runtime.log_config import get_logger +from sandbox_runtime.process_output import spawn_owned_subprocess, terminate_owned_subprocess + +COMMAND_TIMEOUT_SECONDS = 120 +ROOTFS_TAR = "/opt/openinspect/docker-smoke/rootfs.tar" +ROOTFS_IMAGE = "openinspect/docker-smoke-rootfs:local" +BUILT_IMAGE = "openinspect/docker-smoke:local" +COMPOSE_PROJECT = "openinspect-docker-smoke" +MARKER = "compose-network-ok" + + +async def run_command(*command: str, expect: bytes | None = None) -> bytes: + process = await spawn_owned_subprocess( + asyncio.create_subprocess_exec( + *command, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.DEVNULL, + start_new_session=True, + ) + ) + try: + async with asyncio.timeout(COMMAND_TIMEOUT_SECONDS): + stdout, _ = await process.communicate() + except BaseException: + await terminate_owned_subprocess(process) + raise + if process.returncode != 0: + raise RuntimeError(f"Docker verification command failed: {command[1]}") + if expect is not None and stdout.strip() != expect: + raise RuntimeError(f"Docker verification command returned unexpected output: {command[1]}") + return stdout + + +def write_fixtures(workdir: Path) -> None: + (workdir / "Dockerfile").write_text( + f"FROM {ROOTFS_IMAGE}\nRUN mkdir -p /www && printf {MARKER} > /www/marker\n" + ) + (workdir / "compose.yaml").write_text( + "services:\n" + " server:\n" + f" image: {BUILT_IMAGE}\n" + " pull_policy: never\n" + ' command: ["httpd", "-f", "-p", "8080", "-h", "/www"]\n' + " client:\n" + f" image: {BUILT_IMAGE}\n" + " pull_policy: never\n" + " depends_on: [server]\n" + ' command: ["sh", "-c", "for i in 1 2 3 4 5 6 7 8 9 10; do ' + f'wget -qO- http://server:8080/marker | grep -qx {MARKER} && exit 0; sleep 1; done; exit 1"]\n' + ) + + +async def verify_workloads(workdir: Path) -> None: + await run_command("docker", "info", "--format", "{{.Driver}}", expect=b"overlay2") + await run_command("docker", "import", ROOTFS_TAR, ROOTFS_IMAGE) + await run_command( + "docker", "buildx", "build", "--pull=false", "--load", "--tag", BUILT_IMAGE, str(workdir) + ) + await run_command( + "docker", + "run", + "--pull=never", + "--rm", + BUILT_IMAGE, + "cat", + "/www/marker", + expect=MARKER.encode(), + ) + await run_command( + "docker", + "compose", + "--project-name", + COMPOSE_PROJECT, + "-f", + str(workdir / "compose.yaml"), + "up", + "--pull", + "never", + "--abort-on-container-exit", + "--exit-code-from", + "client", + ) + + +async def cleanup(workdir: Path) -> list[Exception]: + """Best-effort removal of everything the checks created; failures are returned, not raised.""" + failures: list[Exception] = [] + for command in ( + ( + "docker", + "compose", + "--project-name", + COMPOSE_PROJECT, + "-f", + str(workdir / "compose.yaml"), + "down", + "--volumes", + "--remove-orphans", + ), + ("docker", "image", "rm", "--force", BUILT_IMAGE, ROOTFS_IMAGE), + ): + try: + await run_command(*command) + except Exception as error: + failures.append(error) + return failures + + +async def main() -> int: + service = DockerService(get_logger("docker-smoke")) + await service.start() + try: + with tempfile.TemporaryDirectory() as directory: + workdir = Path(directory) + write_fixtures(workdir) + try: + await verify_workloads(workdir) + except BaseException: + # The verification failure is the diagnosis; cleanup errors are secondary. + await cleanup(workdir) + raise + failures = await cleanup(workdir) + if failures: + raise RuntimeError("Docker verification cleanup failed") from failures[0] + # The build path stops the daemon the same way before a snapshot. + await service.prepare_for_snapshot() + finally: + await service.stop() + return 0 + + +if __name__ == "__main__": + if os.geteuid() != 0: + print("Docker verification must run as root", file=sys.stderr) + sys.exit(1) + sys.exit(asyncio.run(main())) diff --git a/packages/sandbox-runtime/src/sandbox_runtime/bridge.py b/packages/sandbox-runtime/src/sandbox_runtime/bridge.py index 91f146bfd1..80693380d3 100644 --- a/packages/sandbox-runtime/src/sandbox_runtime/bridge.py +++ b/packages/sandbox-runtime/src/sandbox_runtime/bridge.py @@ -46,9 +46,11 @@ from .boot_attach import RECONNECT_BACKOFF_BASE, RECONNECT_MAX_DELAY_SECONDS, BootAttach from .constants import ( BRIDGE_FATAL_ERROR_FILE_PATH, + DOCKER_ENABLED_ENV_VAR, REPO_MANIFEST_FILE_PATH, ) from .diff_capture import ControlPlaneDiffClient, SessionDiffRefreshWorker +from .docker_control import request as request_docker_preparation from .event_forwarder import BufferedEventForwarder from .git_signing import GitSigningError, GitSigningRuntime from .harness import ( @@ -882,6 +884,8 @@ async def contain_activity(deadline: float) -> bool: async def persist_session() -> None: await self._persist_rotated_session_id(self._require_harness(), strict=True) + if os.environ.get(DOCKER_ENABLED_ENV_VAR) == "true": + await request_docker_preparation("prepare") result = await self.shutdown_preparation.prepare( cmd, diff --git a/packages/sandbox-runtime/src/sandbox_runtime/constants.py b/packages/sandbox-runtime/src/sandbox_runtime/constants.py index 48227da6f3..633f2b0916 100644 --- a/packages/sandbox-runtime/src/sandbox_runtime/constants.py +++ b/packages/sandbox-runtime/src/sandbox_runtime/constants.py @@ -3,6 +3,10 @@ # Provider-selected directory for standalone runtime commands. OpenComputer uses # the sandbox user's bin directory; providers with writable images use /usr/local/bin. BIN_INSTALL_DIR_ENV_VAR = "OPENINSPECT_BIN_INSTALL_DIR" + +# Trusted launch signal: the provider sets it to "true" only for a sandbox it +# launched on a Docker-capable runtime. User env vars can never set it. +DOCKER_ENABLED_ENV_VAR = "OPENINSPECT_DOCKER_ENABLED" DEFAULT_BIN_INSTALL_DIR = "/usr/local/bin" # Sandbox lifetime and the env contract used to pass it to the bridge. diff --git a/packages/sandbox-runtime/src/sandbox_runtime/docker_control.py b/packages/sandbox-runtime/src/sandbox_runtime/docker_control.py new file mode 100644 index 0000000000..6c1294759a --- /dev/null +++ b/packages/sandbox-runtime/src/sandbox_runtime/docker_control.py @@ -0,0 +1,79 @@ +"""Local, terminal Docker preparation owned by the runtime supervisor. + +VM snapshots are terminal: only acknowledged preparation permits capture. +The provider retires the VM after capture; there is no implicit container restart. +""" + +from __future__ import annotations + +import asyncio +import sys +from pathlib import Path +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from .docker_service import DockerService + +SOCKET_PATH = "/tmp/openinspect-docker-control.sock" +CONTROL_TIMEOUT_SECONDS = 45 + + +class DockerControl: + def __init__(self, service: DockerService, path: str = SOCKET_PATH) -> None: + self.service = service + self.path = path + self.prepared = False + self._lock = asyncio.Lock() + self._server: asyncio.Server | None = None + + async def start(self) -> None: + Path(self.path).unlink(missing_ok=True) + self._server = await asyncio.start_unix_server(self._handle, path=self.path) + Path(self.path).chmod(0o600) + + async def stop(self) -> None: + if self._server: + self._server.close() + await self._server.wait_closed() + Path(self.path).unlink(missing_ok=True) + + async def _handle(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + try: + async with asyncio.timeout(CONTROL_TIMEOUT_SECONDS): + command = await reader.readline() + async with self._lock: + if command == b"prepare\n" and not self.prepared: + await self.service.prepare_for_snapshot() + self.prepared = True + result = ( + b"prepared\n" + if self.prepared and command in (b"prepare\n", b"status\n") + else b"not_prepared\n" + ) + writer.write(result) + await writer.drain() + except (Exception, asyncio.CancelledError): + # No acknowledgement is a failure, never permission to capture. + pass + finally: + writer.close() + await writer.wait_closed() + + +async def request(command: str, path: str = SOCKET_PATH) -> None: + async with asyncio.timeout(CONTROL_TIMEOUT_SECONDS): + reader, writer = await asyncio.open_unix_connection(path) + try: + writer.write((command + "\n").encode()) + await writer.drain() + if await reader.readline() != b"prepared\n": + raise RuntimeError("Docker snapshot requires confirmed shutdown preparation") + finally: + writer.close() + await writer.wait_closed() + + +if __name__ == "__main__": + if len(sys.argv) != 2 or sys.argv[1] not in ("status", "prepare"): + raise SystemExit(2) + asyncio.run(request(sys.argv[1])) diff --git a/packages/sandbox-runtime/src/sandbox_runtime/docker_service.py b/packages/sandbox-runtime/src/sandbox_runtime/docker_service.py new file mode 100644 index 0000000000..2a3c8277e4 --- /dev/null +++ b/packages/sandbox-runtime/src/sandbox_runtime/docker_service.py @@ -0,0 +1,193 @@ +"""Foreground-supervised Docker daemon for Docker-enabled sandboxes. + +The runtime supervisor owns ``dockerd`` as an ordinary child process: it is +started before repository hooks, watched for the life of the session, stopped +cleanly before an image build reports success, and reaped on shutdown. Nothing +backgrounds or detaches it, and no remote exec ever owns it. + +Raw daemon and probe output is deliberately not forwarded into the structured +log: registry errors and container metadata can carry repository secrets. The +daemon writes to a log file inside the sandbox for users; the runtime emits +only fixed lifecycle events. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import os +import signal +from pathlib import Path +from typing import Any + +from .process_output import ( + finish_cancellation_cleanup, + spawn_owned_subprocess, + terminate_owned_subprocess, + wait_for_process_exit, +) + +DOCKER_SOCKET = "unix:///var/run/docker.sock" +DOCKER_LOG_PATH = "/var/log/dockerd.log" +DOCKER_START_TIMEOUT_SECONDS = 60.0 +DOCKER_PROBE_TIMEOUT_SECONDS = 5.0 +DOCKER_PROBE_INTERVAL_SECONDS = 0.2 +DOCKER_STOP_TIMEOUT_SECONDS = 30.0 + + +class DockerService: + """Own one ``dockerd`` process from start through clean stop.""" + + def __init__( + self, + log: Any, + *, + start_timeout_seconds: float = DOCKER_START_TIMEOUT_SECONDS, + stop_timeout_seconds: float = DOCKER_STOP_TIMEOUT_SECONDS, + log_path: str = DOCKER_LOG_PATH, + ) -> None: + self.log = log + self.start_timeout_seconds = start_timeout_seconds + self.stop_timeout_seconds = stop_timeout_seconds + self.log_path = log_path + self._process: asyncio.subprocess.Process | None = None + self._stopping = False + + @property + def stopping(self) -> bool: + """Whether the last exit was requested, as opposed to an unexpected daemon death.""" + return self._stopping + + async def start(self) -> None: + """Start the daemon and wait, under a deadline, until ``docker info`` succeeds.""" + if self._process is not None: + raise RuntimeError("Docker service already started") + self._stopping = False + self._process = await self._spawn_daemon() + try: + async with asyncio.timeout(self.start_timeout_seconds): + await self._wait_until_ready() + except TimeoutError: + await self.stop() + raise RuntimeError( + "Required Docker daemon did not become ready before its startup deadline" + ) from None + except BaseException: + await self.stop() + raise + self.log.info("docker.ready") + + async def _spawn_daemon(self) -> asyncio.subprocess.Process: + # The daemon's own output goes to a file the user can read inside the + # sandbox; a closed pipe must never be what stops the daemon. + daemon_log = open(self.log_path, "ab") # noqa: SIM115 - handed to the child + try: + return await spawn_owned_subprocess( + asyncio.create_subprocess_exec( + "dockerd", + "--host", + DOCKER_SOCKET, + stdin=asyncio.subprocess.DEVNULL, + stdout=daemon_log, + stderr=asyncio.subprocess.STDOUT, + start_new_session=True, + ) + ) + finally: + daemon_log.close() + + async def _wait_until_ready(self) -> None: + process = self._process + assert process is not None + while process.returncode is None: + if await self._probe() and process.returncode is None: + return + await asyncio.sleep(DOCKER_PROBE_INTERVAL_SECONDS) + raise RuntimeError("Required Docker daemon exited during startup") + + async def _probe(self) -> bool: + """One bounded ``docker info``; its output is discarded, only the exit code counts.""" + probe = await spawn_owned_subprocess( + asyncio.create_subprocess_exec( + "docker", + "--host", + DOCKER_SOCKET, + "info", + env={ + "PATH": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "HOME": "/root", + }, + stdin=asyncio.subprocess.DEVNULL, + stdout=asyncio.subprocess.DEVNULL, + stderr=asyncio.subprocess.DEVNULL, + start_new_session=True, + ) + ) + try: + async with asyncio.timeout(DOCKER_PROBE_TIMEOUT_SECONDS): + return await wait_for_process_exit(probe) == 0 + except TimeoutError: + return False + finally: + if probe.returncode is None: + cleanup = asyncio.create_task(terminate_owned_subprocess(probe)) + await finish_cancellation_cleanup(cleanup) + + async def wait(self) -> int: + """Block until the daemon exits and return its exit code.""" + process = self._process + if process is None: + raise RuntimeError("Docker service is not running") + return await wait_for_process_exit(process) + + async def prepare_for_snapshot(self) -> None: + """Stop the daemon cleanly so an image build can be snapshotted. + + Only the daemon leader is signalled: Docker must order container and + containerd shutdown itself before its own clean exit, so the whole + group is not killed. A non-zero exit, a timeout, or a daemon that was + already gone is a failed preparation, never a successful build. + """ + process = self._process + if process is None or process.returncode is not None: + raise RuntimeError("Required Docker daemon exited before build preparation") + self._stopping = True + process.send_signal(signal.SIGTERM) + try: + async with asyncio.timeout(self.stop_timeout_seconds): + if await wait_for_process_exit(process) != 0: + raise RuntimeError("Docker build preparation did not stop cleanly") + except TimeoutError: + await self.stop() + raise RuntimeError( + "Docker build preparation exceeded its clean shutdown deadline" + ) from None + # A clean daemon exit means it already stopped containerd and BuildKit; + # anything still alive in the group is a straggler, not a dependency. + with contextlib.suppress(ProcessLookupError): + os.killpg(process.pid, signal.SIGKILL) + self._process = None + # A reusable image must never include secret-bearing daemon diagnostics. + Path(self.log_path).write_bytes(b"") + self.log.info("docker.prepared") + + async def stop(self) -> None: + """Bounded graceful termination, then reap every owned process.""" + process = self._process + self._process = None + self._stopping = True + if process is None: + return + + async def terminate() -> None: + if process.returncode is None: + with contextlib.suppress(ProcessLookupError): + process.send_signal(signal.SIGTERM) + with contextlib.suppress(TimeoutError): + async with asyncio.timeout(self.stop_timeout_seconds): + await wait_for_process_exit(process) + await terminate_owned_subprocess(process) + + cleanup = asyncio.create_task(terminate()) + await finish_cancellation_cleanup(cleanup) + self.log.info("docker.stopped") diff --git a/packages/sandbox-runtime/src/sandbox_runtime/entrypoint.py b/packages/sandbox-runtime/src/sandbox_runtime/entrypoint.py index 035e53ca7b..a018007288 100644 --- a/packages/sandbox-runtime/src/sandbox_runtime/entrypoint.py +++ b/packages/sandbox-runtime/src/sandbox_runtime/entrypoint.py @@ -15,6 +15,7 @@ from .claude_stager import ClaudeStager, isolated_claude_config_dir, resolve_claude_config_dir from .code_server import CodeServer from .constants import VNC_DISPLAY, VNC_PASSWORD_ENV_VAR +from .docker_service import DockerService from .harness.base import HarnessId, HarnessProcessOwner from .image_build_context_start import ( IMAGE_BUILD_CONTEXT_START_ARGUMENT, @@ -146,6 +147,7 @@ def build_supervisor(shutdown_event: asyncio.Event) -> SandboxSupervisor: shutdown_event, log, boot_events=warnings, + docker_service=DockerService(log) if config.docker_enabled else None, ) diff --git a/packages/sandbox-runtime/src/sandbox_runtime/image_build_context_start.py b/packages/sandbox-runtime/src/sandbox_runtime/image_build_context_start.py index 8972c91763..50e3471ebb 100644 --- a/packages/sandbox-runtime/src/sandbox_runtime/image_build_context_start.py +++ b/packages/sandbox-runtime/src/sandbox_runtime/image_build_context_start.py @@ -32,7 +32,7 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Protocol -from .constants import IMAGE_BUILD_EXECUTION_TIMEOUT_ENV_VAR +from .constants import DOCKER_ENABLED_ENV_VAR, IMAGE_BUILD_EXECUTION_TIMEOUT_ENV_VAR from .log_config import StructuredLogger, get_logger from .repo_image_callback import ( BUILD_ID_ENV, @@ -101,6 +101,8 @@ "VCS_HOST", "VCS_CLONE_USERNAME", VCS_CLONE_TOKEN_ENV, + # Trusted runtime signal (provider-owned). + DOCKER_ENABLED_ENV_VAR, } ) diff --git a/packages/sandbox-runtime/src/sandbox_runtime/runtime_config.py b/packages/sandbox-runtime/src/sandbox_runtime/runtime_config.py index 4733d92ded..92214a6071 100644 --- a/packages/sandbox-runtime/src/sandbox_runtime/runtime_config.py +++ b/packages/sandbox-runtime/src/sandbox_runtime/runtime_config.py @@ -11,6 +11,7 @@ from typing import Any from urllib.parse import urlsplit +from .constants import DOCKER_ENABLED_ENV_VAR from .harness.base import HarnessId, parse_harness_id @@ -110,6 +111,9 @@ class RuntimeConfig: session_config: Mapping[str, Any] workspace_path: Path repo_path: Path + # Set by the provider, never by user configuration: this sandbox runs on a + # Docker-capable runtime and must own a daemon before repository hooks. + docker_enabled: bool = False @classmethod def from_env( @@ -137,6 +141,7 @@ def from_env( session_config=session_config, workspace_path=workspace_path, repo_path=repo_path, + docker_enabled=environment.get(DOCKER_ENABLED_ENV_VAR) == "true", ) @property diff --git a/packages/sandbox-runtime/src/sandbox_runtime/supervisor.py b/packages/sandbox-runtime/src/sandbox_runtime/supervisor.py index c8d0c05bbb..8a0928cd52 100644 --- a/packages/sandbox-runtime/src/sandbox_runtime/supervisor.py +++ b/packages/sandbox-runtime/src/sandbox_runtime/supervisor.py @@ -16,6 +16,7 @@ BRIDGE_FATAL_ERROR_FILE_PATH, IMAGE_BUILD_EXECUTION_TIMEOUT_ENV_VAR, ) +from .docker_control import DockerControl from .harness.base import DETERMINISTIC_FAILURE_EXIT_CODE from .repo_image_callback import RepoImageBuildCallback from .runtime_config import BootMode, RuntimeConfig @@ -28,6 +29,7 @@ from .boot_events import BootEventLog from .browser_desktop import BrowserDesktop from .code_server import CodeServer + from .docker_service import DockerService from .harness.base import HarnessProcessOwner from .managed_skills import ManagedSkillsMaterializer from .repository_boot import RepositoryBoot, RepositoryBootResult @@ -66,9 +68,16 @@ def __init__( log: Any, *, boot_events: BootEventLog | None = None, + docker_service: DockerService | None = None, ) -> None: self.config = config self.repository_boot = repository_boot + # Present only for Docker-enabled sandboxes: started before repository + # hooks, watched for the whole session, stopped last. + self.docker_service = docker_service + self.docker_control = DockerControl(docker_service) if docker_service is not None else None + self._docker_watch_task: asyncio.Task[None] | None = None + self._docker_watch_failure: BaseException | None = None # The boot-events channel the bridge relays; the repository boot # writes its own phases and warnings through the same log. self.boot_events: BootEventLog = ( @@ -374,6 +383,45 @@ async def _watch_bridge_during_boot(self) -> None: self._bridge_watch_failure = error self.shutdown_event.set() + async def _start_docker(self) -> None: + """Start the owned daemon; only called when the trusted launch config requires Docker.""" + if self.docker_service is None: + raise RuntimeError("Required Docker service is not configured") + await self.docker_service.start() + self._docker_watch_task = asyncio.create_task(self._watch_docker()) + if self.docker_control is not None and self.boot_mode is not BootMode.BUILD: + await self.docker_control.start() + + async def _watch_docker(self) -> None: + """An unrequested daemon exit is fatal for as long as Docker is required.""" + service = self.docker_service + assert service is not None + try: + await service.wait() + except asyncio.CancelledError: + raise + except Exception as error: + self.log.error("docker.watch_failed", exc=error) + self._docker_watch_failure = error + self.shutdown_event.set() + return + if service.stopping: + return + self.log.error("docker.exited_unexpectedly") + self._docker_watch_failure = RuntimeError("Required Docker daemon exited unexpectedly") + # Interrupt hooks and the process monitor; the failure is reported + # by ``run`` rather than treated as a requested shutdown. + self.shutdown_event.set() + + async def _stop_docker_watch(self) -> None: + task = self._docker_watch_task + if task is None: + return + self._docker_watch_task = None + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + async def _stop_bridge_watch(self) -> None: task = self._bridge_watch_task if task is None: @@ -446,7 +494,7 @@ def _boot_interruption(self) -> BaseException: A failure is raised as itself so ``run`` reports it fatally; a requested shutdown is a clean end to the boot. """ - failure = self._bridge_watch_failure + failure = self._docker_watch_failure or self._bridge_watch_failure if failure is not None: return failure return BootExecutionCancelled() @@ -478,9 +526,23 @@ async def _run_image_build_execution( timeout_seconds = self._image_build_execution_timeout_seconds() try: async with asyncio.timeout(timeout_seconds): - return await self._run_until_shutdown( + if not self.config.docker_enabled: + return await self._run_until_shutdown( + lambda: self.repository_boot.boot(BootMode.BUILD, expected_tunnel_ports) + ) + # Docker starts before setup hooks, and is stopped cleanly + # before success is reported: the snapshot must hold a + # quiesced data root, never a daemon mid-write. + await self._run_until_shutdown(self._start_docker) + result = await self._run_until_shutdown( lambda: self.repository_boot.boot(BootMode.BUILD, expected_tunnel_ports) ) + assert self.docker_service is not None + await self._run_until_shutdown(self.docker_service.prepare_for_snapshot) + await self._stop_docker_watch() + if self._docker_watch_failure is not None: + raise self._docker_watch_failure + return result except TimeoutError as error: raise RuntimeError( f"image build exceeded its {timeout_seconds}-second execution timeout" @@ -582,6 +644,11 @@ async def run(self, repo_image_callback: RepoImageBuildCallback | None = None) - await self.agent_bridge.start(early_connect=True) self._bridge_watch_task = asyncio.create_task(self._watch_bridge_during_boot()) + if self.config.docker_enabled: + # Docker before the desktop and the repository boot: setup and + # start hooks may run containers. + await self._run_until_shutdown(self._start_docker) + try: await self.browser_desktop.start() except Exception as error: @@ -624,6 +691,10 @@ async def run(self, repo_image_callback: RepoImageBuildCallback | None = None) - if self._bridge_watch_failure is not None: raise self._bridge_watch_failure await self.monitor_processes() + # The Docker watcher runs for the whole session: a daemon that + # died under a working harness ended the session as a failure. + if self._docker_watch_failure is not None: + raise self._docker_watch_failure except BootExecutionCancelled: event = ( "image_build.cancelled" @@ -634,23 +705,39 @@ async def run(self, repo_image_callback: RepoImageBuildCallback | None = None) - return True except Exception as error: self.log.error("supervisor.error", exc=error) - if self.boot_mode is BootMode.BUILD and self.shutdown_event.is_set(): + docker_failed = self._docker_watch_failure is not None + if ( + self.boot_mode is BootMode.BUILD + and self.shutdown_event.is_set() + and not docker_failed + ): self.log.info("image_build.cancelled", reason="shutdown_requested") return True if self.boot_mode is BootMode.BUILD and repo_image_callback: - try: - error_message = str(error) - await self._run_until_shutdown( - lambda: repo_image_callback.report_failure(error_message) - ) - except BootExecutionCancelled: - self.log.info("image_build.cancelled", reason="shutdown_requested") - return True + error_message = str(error) + if docker_failed: + # The watcher set shutdown_event to interrupt hooks; that + # is not a requested cancellation and the failure must + # still reach the control plane. + try: + async with asyncio.timeout(FATAL_ERROR_REPORT_TIMEOUT_SECONDS): + await repo_image_callback.report_failure(error_message) + except Exception: + self.log.error("image_build.failure_report_failed") + else: + try: + await self._run_until_shutdown( + lambda: repo_image_callback.report_failure(error_message) + ) + except BootExecutionCancelled: + self.log.info("image_build.cancelled", reason="shutdown_requested") + return True await self._report_fatal_error( str(error), error if isinstance(error, BootPhaseError) else None ) return False finally: + await self._stop_docker_watch() await self._stop_bridge_watch() await self.shutdown() return True @@ -661,6 +748,8 @@ def request_shutdown(self, sig: signal.Signals) -> None: async def shutdown(self) -> None: self.log.info("supervisor.shutdown_start") + if self.docker_control is not None: + await self.docker_control.stop() if self._desktop_restart_task and not self._desktop_restart_task.done(): self._desktop_restart_task.cancel() await asyncio.gather(self._desktop_restart_task, return_exceptions=True) @@ -670,4 +759,7 @@ async def shutdown(self) -> None: await self.code_server.stop() await self.browser_desktop.stop() await self.harness_process.stop() + # User containers outlive the harness that drove them, never the reverse. + if self.docker_service is not None: + await self.docker_service.stop() self.log.info("supervisor.shutdown_complete") diff --git a/packages/sandbox-runtime/tests/test_docker_control.py b/packages/sandbox-runtime/tests/test_docker_control.py new file mode 100644 index 0000000000..5b2c68fc0e --- /dev/null +++ b/packages/sandbox-runtime/tests/test_docker_control.py @@ -0,0 +1,46 @@ +import asyncio +import tempfile +from unittest.mock import AsyncMock, Mock + +import pytest + +from sandbox_runtime.docker_control import DockerControl, request + + +@pytest.fixture +def socket_path(): + with tempfile.TemporaryDirectory(prefix="oi-docker-", dir="/tmp") as directory: + yield directory + "/control.sock" + + +@pytest.mark.asyncio +async def test_preparation_is_acknowledged_only_after_clean_stop(socket_path): + service = Mock(prepare_for_snapshot=AsyncMock()) + # Short paths also work on macOS's small Unix-domain path limit. + path = socket_path + control = DockerControl(service, path) + await control.start() + try: + with pytest.raises(RuntimeError, match="confirmed shutdown"): + await request("status", path) + await asyncio.gather(request("prepare", path), request("prepare", path)) + service.prepare_for_snapshot.assert_awaited_once() + await request("status", path) + finally: + await control.stop() + + +@pytest.mark.asyncio +async def test_failed_preparation_never_acknowledges_capture(socket_path): + service = Mock(prepare_for_snapshot=AsyncMock(side_effect=RuntimeError("stop failed"))) + path = socket_path + control = DockerControl(service, path) + await control.start() + try: + with pytest.raises(RuntimeError): + await request("prepare", path) + assert not control.prepared + with pytest.raises(RuntimeError): + await request("status", path) + finally: + await control.stop() diff --git a/packages/sandbox-runtime/tests/test_docker_service.py b/packages/sandbox-runtime/tests/test_docker_service.py new file mode 100644 index 0000000000..91f8ace204 --- /dev/null +++ b/packages/sandbox-runtime/tests/test_docker_service.py @@ -0,0 +1,217 @@ +"""DockerService owns dockerd from start through clean stop, deterministically. + +The fakes implement a real readiness handshake: the fake daemon installs its +SIGTERM handler and only then publishes a ready marker, and the fake +``docker info`` succeeds only once that marker exists. The service therefore +cannot observe readiness before the daemon can honor a clean stop, which is +also the production contract (``docker info`` succeeds only once the daemon +serves the socket). +""" + +from __future__ import annotations + +import asyncio +import os +import sys +from dataclasses import dataclass, field + +import pytest + +from sandbox_runtime import docker_service as docker_module +from sandbox_runtime.docker_service import DockerService + + +@dataclass +class FakeProcesses: + ready_marker: str + daemon_exit: int = 0 + probe_delay: float = 0.0 + fail_probe: bool = False + ignore_sigterm: bool = False + children: list[asyncio.subprocess.Process] = field(default_factory=list) + spawns: list[tuple[str, ...]] = field(default_factory=list) + + def daemon_program(self) -> str: + handler = ( + "signal.SIG_IGN" if self.ignore_sigterm else f"lambda *_: sys.exit({self.daemon_exit})" + ) + return ( + "import signal, sys, time, pathlib; " + f"signal.signal(signal.SIGTERM, {handler}); " + f"pathlib.Path({self.ready_marker!r}).write_text('ready'); " + "time.sleep(300)" + ) + + def probe_program(self) -> str: + return ( + "import sys, time, pathlib; " + f"time.sleep({self.probe_delay}); " + f"sys.exit(1 if {self.fail_probe} or not pathlib.Path({self.ready_marker!r}).exists() else 0)" + ) + + +@pytest.fixture +def processes(monkeypatch, tmp_path): + fakes = FakeProcesses(ready_marker=str(tmp_path / "dockerd.ready")) + real_spawn = asyncio.create_subprocess_exec + + async def spawn(command, *args, **kwargs): + fakes.spawns.append((command, *args)) + assert kwargs.get("start_new_session") is True + if command == "dockerd": + program = fakes.daemon_program() + elif command == "docker": + assert kwargs["env"] == { + "PATH": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "HOME": "/root", + } + program = fakes.probe_program() + else: # pragma: no cover - the service spawns nothing else + raise AssertionError(command) + process = await real_spawn(sys.executable, "-c", program, **kwargs) + fakes.children.append(process) + return process + + monkeypatch.setattr(docker_module.asyncio, "create_subprocess_exec", spawn) + monkeypatch.setattr(docker_module, "DOCKER_PROBE_INTERVAL_SECONDS", 0.01) + yield fakes + for child in fakes.children: + if child.returncode is None: + child.kill() + + +def _service(tmp_path, **kwargs) -> DockerService: + class Log: + def __init__(self): + self.events: list[str] = [] + + def info(self, event, **_fields): + self.events.append(event) + + def error(self, event, **_fields): + self.events.append(event) + + service = DockerService(Log(), log_path=str(tmp_path / "dockerd.log"), **kwargs) + return service + + +def _group_gone(process: asyncio.subprocess.Process) -> bool: + try: + os.killpg(process.pid, 0) + except ProcessLookupError: + return True + return False + + +async def _until(predicate, timeout: float = 10.0) -> None: + """Bounded wait: a regression fails the test instead of hanging it.""" + async with asyncio.timeout(timeout): + while not predicate(): + await asyncio.sleep(0.01) + + +async def test_ready_then_clean_preparation_leaves_no_owned_process(processes, tmp_path): + service = _service(tmp_path) + + await service.start() + daemon = processes.children[0] + assert daemon.returncode is None + assert processes.spawns[0] == ("dockerd", "--host", "unix:///var/run/docker.sock") + assert processes.spawns[1] == ("docker", "--host", "unix:///var/run/docker.sock", "info") + + await service.prepare_for_snapshot() + + assert daemon.returncode == 0 + assert _group_gone(daemon) + assert service.stopping is True + await service.stop() + assert all(child.returncode is not None for child in processes.children) + + +async def test_startup_deadline_has_its_own_diagnostic_and_reaps_the_daemon(processes, tmp_path): + processes.fail_probe = True + service = _service(tmp_path, start_timeout_seconds=0.3) + + with pytest.raises(RuntimeError, match="startup deadline"): + await service.start() + + assert all(child.returncode is not None for child in processes.children) + assert all(_group_gone(child) for child in processes.children) + + +async def test_daemon_exit_during_startup_is_reported(processes, tmp_path): + processes.fail_probe = True + service = _service(tmp_path, start_timeout_seconds=60) + started = asyncio.create_task(service.start()) + await _until(lambda: bool(processes.children)) + processes.children[0].kill() + + with pytest.raises(RuntimeError, match="exited during startup"): + await started + + +async def test_cancellation_during_probe_reaps_both_process_groups(processes, tmp_path): + processes.probe_delay = 300 + service = _service(tmp_path, start_timeout_seconds=60) + started = asyncio.create_task(service.start()) + await _until(lambda: len(processes.children) >= 2) + + started.cancel() + with pytest.raises(asyncio.CancelledError): + await started + + assert all(child.returncode is not None for child in processes.children) + assert all(_group_gone(child) for child in processes.children) + + +async def test_nonzero_daemon_exit_cannot_be_a_prepared_build(processes, tmp_path): + processes.daemon_exit = 1 + service = _service(tmp_path) + await service.start() + + with pytest.raises(RuntimeError, match="did not stop cleanly"): + await service.prepare_for_snapshot() + + await service.stop() + assert all(child.returncode is not None for child in processes.children) + + +async def test_unexpected_exit_is_observable_and_not_a_requested_stop(processes, tmp_path): + service = _service(tmp_path) + await service.start() + daemon = processes.children[0] + + daemon.kill() + assert await service.wait() != 0 + assert service.stopping is False + + await service.stop() + assert service.stopping is True + + +async def test_daemon_that_ignores_sigterm_is_killed_and_never_a_prepared_build( + processes, tmp_path +): + processes.ignore_sigterm = True + service = _service(tmp_path, stop_timeout_seconds=0.3) + await service.start() + daemon = processes.children[0] + + with pytest.raises(RuntimeError, match="clean shutdown deadline"): + await service.prepare_for_snapshot() + + assert daemon.returncode is not None + assert _group_gone(daemon) + + +async def test_preparation_requires_a_running_daemon(processes, tmp_path): + service = _service(tmp_path) + with pytest.raises(RuntimeError, match="exited before build preparation"): + await service.prepare_for_snapshot() + + await service.start() + processes.children[0].kill() + await service.wait() + with pytest.raises(RuntimeError, match="exited before build preparation"): + await service.prepare_for_snapshot() + await service.stop() diff --git a/packages/sandbox-runtime/tests/test_runtime_config.py b/packages/sandbox-runtime/tests/test_runtime_config.py index b1f444baee..52e17aa5a6 100644 --- a/packages/sandbox-runtime/tests/test_runtime_config.py +++ b/packages/sandbox-runtime/tests/test_runtime_config.py @@ -78,3 +78,15 @@ def test_session_config_is_recursively_immutable(): assert isinstance(repositories[0], MappingProxyType) with pytest.raises(TypeError): repositories[0]["repo_name"] = "changed" + + +@pytest.mark.parametrize( + ("value", "expected"), + [(None, False), ("", False), ("false", False), ("1", False), ("TRUE", False), ("true", True)], +) +def test_docker_enabled_is_only_the_exact_trusted_value(value, expected): + environment = {"SANDBOX_ID": "sandbox-1"} + if value is not None: + environment["OPENINSPECT_DOCKER_ENABLED"] = value + + assert RuntimeConfig.from_env(environment).docker_enabled is expected diff --git a/packages/sandbox-runtime/tests/test_supervisor_lifecycle.py b/packages/sandbox-runtime/tests/test_supervisor_lifecycle.py index cb4cd76dca..50db2e99fb 100644 --- a/packages/sandbox-runtime/tests/test_supervisor_lifecycle.py +++ b/packages/sandbox-runtime/tests/test_supervisor_lifecycle.py @@ -1,7 +1,10 @@ import asyncio +import signal from pathlib import Path from unittest.mock import AsyncMock, MagicMock +import pytest + from sandbox_runtime.repository_boot import RepositoryBootResult from sandbox_runtime.runtime_config import BootMode, RuntimeConfig from sandbox_runtime.supervisor import SandboxSupervisor @@ -207,3 +210,236 @@ async def test_code_server_restart_exhaustion_is_nonfatal(tmp_path, monkeypatch) await SandboxSupervisor.monitor_processes(supervisor) supervisor._report_fatal_error.assert_not_awaited() + + +def _docker_service(events, *, prepare_error=None): + service = MagicMock() + service.stopping = False + exited = asyncio.Event() + + async def start(): + events.append("docker:start") + + async def wait(): + await exited.wait() + return 137 + + async def prepare_for_snapshot(): + events.append("docker:prepare") + if prepare_error is not None: + raise prepare_error + service.stopping = True + + async def stop(): + events.append("docker:stop") + service.stopping = True + exited.set() + + service.start = AsyncMock(side_effect=start) + service.wait = AsyncMock(side_effect=wait) + service.prepare_for_snapshot = AsyncMock(side_effect=prepare_for_snapshot) + service.stop = AsyncMock(side_effect=stop) + service.exited = exited + return service + + +def _docker_supervisor(tmp_path, events, monkeypatch, **service_kwargs): + supervisor, repository, *rest = _supervisor(tmp_path, events) + supervisor.config = RuntimeConfig.from_env( + { + "SANDBOX_ID": "sandbox-1", + "REPO_OWNER": "acme", + "REPO_NAME": "repo", + "OPENINSPECT_DOCKER_ENABLED": "true", + }, + workspace_path=tmp_path, + ) + supervisor.docker_service = _docker_service(events, **service_kwargs) + return supervisor, repository, *rest + + +async def test_docker_starts_before_repository_boot_and_stops_last(tmp_path, monkeypatch): + events = [] + supervisor, *_ = _docker_supervisor(tmp_path, events, monkeypatch) + monkeypatch.delenv("IMAGE_BUILD_MODE", raising=False) + monkeypatch.delenv("RESTORED_FROM_SNAPSHOT", raising=False) + monkeypatch.delenv("FROM_REPO_IMAGE", raising=False) + + assert await supervisor.run() is True + + assert events[:3] == ["docker:start", "desktop", "repository:fresh"] + assert events[-1] == "docker:stop" + + +async def test_standard_boot_never_touches_docker(tmp_path, monkeypatch): + events = [] + supervisor, *_ = _supervisor(tmp_path, events) + monkeypatch.delenv("IMAGE_BUILD_MODE", raising=False) + monkeypatch.delenv("RESTORED_FROM_SNAPSHOT", raising=False) + monkeypatch.delenv("FROM_REPO_IMAGE", raising=False) + + assert await supervisor.run() is True + + assert supervisor.docker_service is None + assert not any(event.startswith("docker:") for event in events) + + +async def test_docker_required_but_unconfigured_is_fatal(tmp_path, monkeypatch): + events = [] + supervisor, *_ = _docker_supervisor(tmp_path, events, monkeypatch) + supervisor.docker_service = None + supervisor._report_fatal_error = AsyncMock() + monkeypatch.delenv("IMAGE_BUILD_MODE", raising=False) + + assert await supervisor.run() is False + + supervisor._report_fatal_error.assert_awaited_once() + assert "Docker service is not configured" in supervisor._report_fatal_error.await_args.args[0] + assert "repository:fresh" not in events + + +async def test_build_starts_docker_before_hooks_and_prepares_it_before_success( + tmp_path, monkeypatch +): + events = [] + supervisor, *_ = _docker_supervisor(tmp_path, events, monkeypatch) + monkeypatch.setenv("IMAGE_BUILD_MODE", "true") + callback = MagicMock() + + async def report_success(**_kwargs): + events.append("success") + assert supervisor._docker_watch_task is None + supervisor.shutdown_event.set() + return True + + callback.report_success = AsyncMock(side_effect=report_success) + callback.report_failure = AsyncMock() + + assert await supervisor.run(callback) is True + + assert events == [ + "docker:start", + "repository:build", + "docker:prepare", + "success", + "docker:stop", + ] + + +@pytest.mark.parametrize( + "error", [RuntimeError("did not stop cleanly"), RuntimeError("clean shutdown deadline")] +) +async def test_build_preparation_failure_is_reported_as_a_failed_build( + tmp_path, monkeypatch, error +): + events = [] + supervisor, *_ = _docker_supervisor(tmp_path, events, monkeypatch, prepare_error=error) + monkeypatch.setenv("IMAGE_BUILD_MODE", "true") + supervisor._report_fatal_error = AsyncMock() + callback = MagicMock() + callback.report_success = AsyncMock() + callback.report_failure = AsyncMock() + + assert await supervisor.run(callback) is False + + callback.report_success.assert_not_awaited() + callback.report_failure.assert_awaited_once() + assert callback.report_failure.await_args.args[0] + assert supervisor.docker_service.stop.await_count == 1 + + +async def test_daemon_exit_during_build_hooks_fails_the_build(tmp_path, monkeypatch): + events = [] + supervisor, repository, *_ = _docker_supervisor(tmp_path, events, monkeypatch) + monkeypatch.setenv("IMAGE_BUILD_MODE", "true") + supervisor._report_fatal_error = AsyncMock() + callback = MagicMock() + callback.report_success = AsyncMock() + callback.report_failure = AsyncMock() + + async def boot(_mode, _ports): + events.append("repository:build") + supervisor.docker_service.exited.set() + await asyncio.Event().wait() + + repository.boot = AsyncMock(side_effect=boot) + + assert await supervisor.run(callback) is False + + callback.report_success.assert_not_awaited() + callback.report_failure.assert_awaited_once() + assert "exited unexpectedly" in callback.report_failure.await_args.args[0] + supervisor._report_fatal_error.assert_awaited_once() + + +async def test_daemon_exit_during_session_is_fatal(tmp_path, monkeypatch): + events = [] + supervisor, *_ = _docker_supervisor(tmp_path, events, monkeypatch) + monkeypatch.delenv("IMAGE_BUILD_MODE", raising=False) + supervisor._report_fatal_error = AsyncMock() + + async def monitor(): + supervisor.docker_service.exited.set() + await supervisor.shutdown_event.wait() + + supervisor.monitor_processes = AsyncMock(side_effect=monitor) + + assert await supervisor.run() is False + + supervisor._report_fatal_error.assert_awaited_once() + assert "exited unexpectedly" in supervisor._report_fatal_error.await_args.args[0] + + +async def test_requested_shutdown_during_docker_start_is_not_a_failure(tmp_path, monkeypatch): + events = [] + supervisor, *_ = _docker_supervisor(tmp_path, events, monkeypatch) + monkeypatch.setenv("IMAGE_BUILD_MODE", "true") + callback = MagicMock() + callback.report_success = AsyncMock() + callback.report_failure = AsyncMock() + + async def start(): + supervisor.shutdown_event.set() + await asyncio.Event().wait() + + supervisor.docker_service.start = AsyncMock(side_effect=start) + + assert await supervisor.run(callback) is True + + callback.report_success.assert_not_awaited() + callback.report_failure.assert_not_awaited() + + +async def test_daemon_exit_during_interactive_boot_is_fatal(tmp_path, monkeypatch): + events = [] + supervisor, repository, *_ = _docker_supervisor(tmp_path, events, monkeypatch) + monkeypatch.delenv("IMAGE_BUILD_MODE", raising=False) + supervisor._report_fatal_error = AsyncMock() + + async def boot(_mode, _ports): + supervisor.docker_service.exited.set() + await asyncio.Event().wait() + + repository.boot = AsyncMock(side_effect=boot) + + assert await supervisor.run() is False + + supervisor._report_fatal_error.assert_awaited_once() + assert "exited unexpectedly" in supervisor._report_fatal_error.await_args.args[0] + + +async def test_requested_shutdown_with_docker_running_is_not_a_failure(tmp_path, monkeypatch): + events = [] + supervisor, *_ = _docker_supervisor(tmp_path, events, monkeypatch) + monkeypatch.delenv("IMAGE_BUILD_MODE", raising=False) + supervisor._report_fatal_error = AsyncMock() + + async def monitor(): + supervisor.request_shutdown(signal.SIGTERM) + + supervisor.monitor_processes = AsyncMock(side_effect=monitor) + + assert await supervisor.run() is True + + supervisor._report_fatal_error.assert_not_awaited() + assert events[-1] == "docker:stop" diff --git a/packages/shared/src/types/integrations.ts b/packages/shared/src/types/integrations.ts index 5b82121d95..81a1e98430 100644 --- a/packages/shared/src/types/integrations.ts +++ b/packages/shared/src/types/integrations.ts @@ -282,6 +282,7 @@ export type SandboxSettings = z.infer; /** Every supported sandbox backend. Keep provider policy exhaustive over this union. */ export const SANDBOX_PROVIDER_NAMES = [ "modal", + "modal-vm", "daytona", "vercel", "opencomputer", @@ -293,6 +294,7 @@ export type SandboxProviderName = (typeof SANDBOX_PROVIDER_NAMES)[number]; const DEFAULT_SANDBOX_SETTING_CAPABILITIES = { resources: true, timeout: true }; const SANDBOX_SETTING_CAPABILITIES = { modal: DEFAULT_SANDBOX_SETTING_CAPABILITIES, + "modal-vm": DEFAULT_SANDBOX_SETTING_CAPABILITIES, daytona: { resources: false, timeout: false }, vercel: DEFAULT_SANDBOX_SETTING_CAPABILITIES, opencomputer: { resources: false, timeout: true }, diff --git a/packages/web/src/lib/sandbox-provider.test.ts b/packages/web/src/lib/sandbox-provider.test.ts index 27f70b0ac7..d467fb9681 100644 --- a/packages/web/src/lib/sandbox-provider.test.ts +++ b/packages/web/src/lib/sandbox-provider.test.ts @@ -83,13 +83,13 @@ describe("sandbox-provider", () => { expect(supportsRepoImages()).toBe(true); }); - it("supports e2b with repo images", async () => { + it.each(["e2b", "modal-vm"])("supports %s with repo images", async (backend) => { delete process.env.NEXT_PUBLIC_SANDBOX_PROVIDER; - process.env.SANDBOX_PROVIDER = "e2b"; + process.env.SANDBOX_PROVIDER = backend; const { getPublicSandboxProvider, supportsRepoImages } = await loadProvider(); - expect(getPublicSandboxProvider()).toBe("e2b"); + expect(getPublicSandboxProvider()).toBe(backend); expect(supportsRepoImages()).toBe(true); }); @@ -99,7 +99,7 @@ describe("sandbox-provider", () => { // The message is the only copy of this sentence in the web app; the routes // import it rather than restating the provider list. expect(REPO_IMAGES_UNSUPPORTED_MESSAGE).toBe( - "Image builds are only available when SANDBOX_PROVIDER=modal, vercel, opencomputer, e2b, or daytona" + "Image builds are only available when SANDBOX_PROVIDER=modal, modal-vm, vercel, opencomputer, e2b, or daytona" ); for (const provider of getRepoImageProviders()) { expect(REPO_IMAGES_UNSUPPORTED_MESSAGE).toContain(provider); diff --git a/packages/web/src/lib/sandbox-provider.ts b/packages/web/src/lib/sandbox-provider.ts index 3b04aced61..c6e786d106 100644 --- a/packages/web/src/lib/sandbox-provider.ts +++ b/packages/web/src/lib/sandbox-provider.ts @@ -16,7 +16,14 @@ export type PublicSandboxProvider = SandboxProviderName; * IMAGE_BUILD_PROVIDERS in the control plane's image-builds/provider-policy.ts — * a provider gains image-build support in both places or neither. */ -const REPO_IMAGE_PROVIDERS = ["modal", "vercel", "opencomputer", "e2b", "daytona"] as const; +const REPO_IMAGE_PROVIDERS = [ + "modal", + "modal-vm", + "vercel", + "opencomputer", + "e2b", + "daytona", +] as const; /** * The single 501 body every image-build route answers with when the deployment's diff --git a/terraform/environments/production/locals.tf b/terraform/environments/production/locals.tf index d3c65896e1..711f6e3994 100644 --- a/terraform/environments/production/locals.tf +++ b/terraform/environments/production/locals.tf @@ -1,6 +1,6 @@ locals { name_suffix = var.deployment_name - use_modal_backend = var.sandbox_provider == "modal" + use_modal_backend = contains(["modal", "modal-vm"], var.sandbox_provider) use_daytona_backend = var.sandbox_provider == "daytona" use_vercel_backend = var.sandbox_provider == "vercel" use_opencomputer_backend = var.sandbox_provider == "opencomputer" diff --git a/terraform/environments/production/modal.tf b/terraform/environments/production/modal.tf index d330ab4fa8..ba266b0b05 100644 --- a/terraform/environments/production/modal.tf +++ b/terraform/environments/production/modal.tf @@ -26,6 +26,7 @@ module "modal_app" { deploy_path = "${var.project_root}/packages/modal-infra" deploy_module = "deploy" source_hash = data.external.modal_source_hash[0].result.hash + build_vm_image = var.sandbox_provider == "modal-vm" secrets = [ { diff --git a/terraform/environments/production/terraform.tfvars.example b/terraform/environments/production/terraform.tfvars.example index b6be1fe174..cd869a7ed8 100644 --- a/terraform/environments/production/terraform.tfvars.example +++ b/terraform/environments/production/terraform.tfvars.example @@ -56,13 +56,13 @@ cloudflare_worker_subdomain = "" # vercel_team_id = "" # e.g., "team_xxxxxxxxxxxxxxxxxxxx" # Modal API token -# Only required when sandbox_provider = "modal" +# Only required when sandbox_provider = "modal" or "modal-vm" # Create at: https://modal.com/settings modal_token_id = "" modal_token_secret = "" # Modal workspace, environment, and endpoint web suffix -# Only required when sandbox_provider = "modal" +# Only required when sandbox_provider = "modal" or "modal-vm" # workspace: your Modal username (e.g., "myworkspace") # environment: Modal environment name used by the CLI # environment_web_suffix: Modal web suffix used in endpoint URLs: @@ -276,7 +276,7 @@ repo_secrets_encryption_key = "" provider_accounts_encryption_key = "" # Modal API secret (for control plane -> Modal authentication) -# Only required when sandbox_provider = "modal" +# Only required when sandbox_provider = "modal" or "modal-vm" # Generate with: openssl rand -hex 32 modal_api_secret = "" @@ -293,6 +293,7 @@ nextauth_secret = "" # - "vercel": Vercel Sandbox API with filesystem snapshot restore # - "opencomputer": OpenComputer declarative-template sandboxes with hibernate/resume # - "e2b": direct E2B REST API integration with pause/resume +# Select "modal-vm" for Docker-capable VMs (deployment-wide; see docs/MODAL_DOCKER.md). sandbox_provider = "modal" # Platform for the web app deployment diff --git a/terraform/environments/production/tests/modal_backends.tftest.hcl b/terraform/environments/production/tests/modal_backends.tftest.hcl new file mode 100644 index 0000000000..05b207fcd7 --- /dev/null +++ b/terraform/environments/production/tests/modal_backends.tftest.hcl @@ -0,0 +1,70 @@ +mock_provider "cloudflare" {} +mock_provider "external" { + mock_data "external" { + defaults = { + result = { + hash = "test-source-hash" + } + } + } +} +mock_provider "local" {} +mock_provider "null" {} +mock_provider "random" {} +mock_provider "vercel" {} + +variables { + cloudflare_api_token = "test-cloudflare-token" + cloudflare_account_id = "test-account" + cloudflare_worker_subdomain = "test-account" + github_app_id = "1" + github_app_private_key = "test-private-key" + github_app_installation_id = "1" + anthropic_api_key = "test-anthropic-key" + token_encryption_key = "test-token-key" + repo_secrets_encryption_key = "test-repo-key" + nextauth_secret = "test-browser-auth-secret-with-32-characters" + deployment_name = "daytona-snapshot-memory-test" + + sandbox_provider = "modal" + modal_token_id = "test-token" + modal_token_secret = "test-secret" + modal_workspace = "test-workspace" + modal_api_secret = "test-api-secret" + + web_platform = "cloudflare" + project_root = "../../../" + enable_github_bot = false + enable_slack_bot = false + enable_linear_bot = false + + github_client_id = "github-id" + github_client_secret = "github-secret" + allowed_users = "octocat" +} + + +run "gvisor_uses_shared_modal_infrastructure" { + command = plan + assert { + condition = local.use_modal_backend && length(module.modal_app) == 1 && length(data.external.modal_source_hash) == 1 + error_message = "Standard Modal must provision its shared infrastructure." + } +} +run "vm_uses_shared_modal_infrastructure" { + command = plan + variables { sandbox_provider = "modal-vm" } + assert { + condition = local.use_modal_backend && length(module.modal_app) == 1 && length(data.external.modal_source_hash) == 1 + error_message = "Modal VM must provision the same Modal module and credentials." + } +} +run "vm_requires_modal_credentials" { + command = plan + variables { + sandbox_provider = "modal-vm" + modal_token_id = "" + } + expect_failures = [var.modal_token_id] +} + diff --git a/terraform/environments/production/variables.tf b/terraform/environments/production/variables.tf index a1a97ca693..7f8b19b3d0 100644 --- a/terraform/environments/production/variables.tf +++ b/terraform/environments/production/variables.tf @@ -59,8 +59,8 @@ variable "modal_token_id" { default = "" validation { - condition = var.sandbox_provider != "modal" || length(var.modal_token_id) > 0 - error_message = "modal_token_id must be set when sandbox_provider = 'modal'." + condition = !contains(["modal", "modal-vm"], var.sandbox_provider) || length(var.modal_token_id) > 0 + error_message = "modal_token_id must be set when sandbox_provider is 'modal' or 'modal-vm'." } } @@ -71,8 +71,8 @@ variable "modal_token_secret" { default = "" validation { - condition = var.sandbox_provider != "modal" || length(var.modal_token_secret) > 0 - error_message = "modal_token_secret must be set when sandbox_provider = 'modal'." + condition = !contains(["modal", "modal-vm"], var.sandbox_provider) || length(var.modal_token_secret) > 0 + error_message = "modal_token_secret must be set when sandbox_provider is 'modal' or 'modal-vm'." } } @@ -82,8 +82,8 @@ variable "modal_workspace" { default = "" validation { - condition = var.sandbox_provider != "modal" || length(var.modal_workspace) > 0 - error_message = "modal_workspace must be set when sandbox_provider = 'modal'." + condition = !contains(["modal", "modal-vm"], var.sandbox_provider) || length(var.modal_workspace) > 0 + error_message = "modal_workspace must be set when sandbox_provider is 'modal' or 'modal-vm'." } } @@ -93,8 +93,8 @@ variable "modal_environment" { default = "main" validation { - condition = var.sandbox_provider != "modal" || (length(trimspace(var.modal_environment)) > 0 && can(regex("^[^:/\\\\]+$", var.modal_environment))) - error_message = "modal_environment must be set and must not contain colons, slashes, or backslashes when sandbox_provider = 'modal'." + condition = !contains(["modal", "modal-vm"], var.sandbox_provider) || (length(trimspace(var.modal_environment)) > 0 && can(regex("^[^:/\\\\]+$", var.modal_environment))) + error_message = "modal_environment must be set and must not contain colons, slashes, or backslashes when sandbox_provider is 'modal' or 'modal-vm'." } } @@ -104,8 +104,8 @@ variable "modal_environment_web_suffix" { default = "" validation { - condition = var.sandbox_provider != "modal" || can(regex("^$|^[a-z0-9-]+$", var.modal_environment_web_suffix)) - error_message = "modal_environment_web_suffix must be empty or contain only lowercase letters, digits, and dashes when sandbox_provider = 'modal'." + condition = !contains(["modal", "modal-vm"], var.sandbox_provider) || can(regex("^$|^[a-z0-9-]+$", var.modal_environment_web_suffix)) + error_message = "modal_environment_web_suffix must be empty or contain only lowercase letters, digits, and dashes when sandbox_provider is 'modal' or 'modal-vm'." } } @@ -443,8 +443,8 @@ variable "modal_api_secret" { default = "" validation { - condition = var.sandbox_provider != "modal" || length(var.modal_api_secret) > 0 - error_message = "modal_api_secret must be set when sandbox_provider = 'modal'." + condition = !contains(["modal", "modal-vm"], var.sandbox_provider) || length(var.modal_api_secret) > 0 + error_message = "modal_api_secret must be set when sandbox_provider is 'modal' or 'modal-vm'." } } @@ -671,7 +671,7 @@ variable "sandbox_provider" { default = "modal" validation { - condition = contains(["modal", "daytona", "vercel", "opencomputer", "e2b"], var.sandbox_provider) + condition = contains(["modal", "modal-vm", "daytona", "vercel", "opencomputer", "e2b"], var.sandbox_provider) error_message = "sandbox_provider must be 'modal', 'daytona', 'vercel', 'opencomputer', or 'e2b'." } } diff --git a/terraform/modules/modal-app/main.tf b/terraform/modules/modal-app/main.tf index 03d840fbbe..e0288314c5 100644 --- a/terraform/modules/modal-app/main.tf +++ b/terraform/modules/modal-app/main.tf @@ -36,7 +36,8 @@ resource "null_resource" "modal_secrets" { resource "null_resource" "modal_deploy" { triggers = { # Re-deploy when source files change - source_hash = var.source_hash + build_vm_image = tostring(var.build_vm_image) + source_hash = var.source_hash # Re-deploy when app name changes app_name = var.app_name # Re-deploy when Modal environment changes @@ -50,12 +51,13 @@ resource "null_resource" "modal_deploy" { interpreter = ["bash"] environment = { - MODAL_TOKEN_ID = var.modal_token_id - MODAL_TOKEN_SECRET = var.modal_token_secret - MODAL_ENVIRONMENT = var.modal_environment - APP_NAME = var.app_name - DEPLOY_PATH = var.deploy_path - DEPLOY_MODULE = var.deploy_module + MODAL_TOKEN_ID = var.modal_token_id + MODAL_TOKEN_SECRET = var.modal_token_secret + MODAL_ENVIRONMENT = var.modal_environment + APP_NAME = var.app_name + DEPLOY_PATH = var.deploy_path + DEPLOY_MODULE = var.deploy_module + BUILD_MODAL_VM_IMAGE = tostring(var.build_vm_image) } } diff --git a/terraform/modules/modal-app/tests/backends.tftest.hcl b/terraform/modules/modal-app/tests/backends.tftest.hcl new file mode 100644 index 0000000000..29fba2895d --- /dev/null +++ b/terraform/modules/modal-app/tests/backends.tftest.hcl @@ -0,0 +1,28 @@ +mock_provider "null" {} +mock_provider "external" {} + +variables { + app_name = "open-inspect" + modal_token_id = "test-token" + modal_token_secret = "test-secret" + workspace = "test-workspace" + deploy_path = "." + source_hash = "test-source" + fetch_app_info = false +} + +run "standard_omits_vm_image" { + command = plan + assert { + condition = null_resource.modal_deploy.triggers.build_vm_image == "false" + error_message = "Standard deployments must not build the paid VM verification image." + } +} +run "vm_image_change_forces_redeploy" { + command = plan + variables { build_vm_image = true } + assert { + condition = null_resource.modal_deploy.triggers.build_vm_image == "true" + error_message = "Selecting VM image provisioning must participate in deployment identity." + } +} diff --git a/terraform/modules/modal-app/variables.tf b/terraform/modules/modal-app/variables.tf index a1b316d360..d35e2bc18c 100644 --- a/terraform/modules/modal-app/variables.tf +++ b/terraform/modules/modal-app/variables.tf @@ -74,3 +74,9 @@ variable "fetch_app_info" { type = bool default = false } + +variable "build_vm_image" { + description = "Build and verify the VM image required by the selected Modal backend." + type = bool + default = false +} From 6f1f90d679909e03a9eb19c2f67d28b8da2a3748 Mon Sep 17 00:00:00 2001 From: Cole Murray Date: Tue, 22 Sep 2026 01:05:57 -0700 Subject: [PATCH 02/22] chore(modal): tighten backend typing and configuration guidance --- packages/control-plane/src/sandbox/provider-factory.ts | 2 +- .../control-plane/src/sandbox/providers/modal-provider.ts | 2 +- packages/modal-infra/src/sandbox/build_session.py | 4 ++-- packages/modal-infra/src/web_api.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/control-plane/src/sandbox/provider-factory.ts b/packages/control-plane/src/sandbox/provider-factory.ts index 54b3b02c15..90615f3e1a 100644 --- a/packages/control-plane/src/sandbox/provider-factory.ts +++ b/packages/control-plane/src/sandbox/provider-factory.ts @@ -24,7 +24,7 @@ import type { Env } from "../types"; function createModalProviderFromEnv(env: Env, backend: "modal" | "modal-vm"): ModalSandboxProvider { if (!env.MODAL_API_SECRET || !env.MODAL_WORKSPACE) { throw new Error( - "MODAL_API_SECRET and MODAL_WORKSPACE are required when SANDBOX_PROVIDER=modal" + `MODAL_API_SECRET and MODAL_WORKSPACE are required when SANDBOX_PROVIDER=${backend}` ); } diff --git a/packages/control-plane/src/sandbox/providers/modal-provider.ts b/packages/control-plane/src/sandbox/providers/modal-provider.ts index 7e2ac05db7..cb6b9384db 100644 --- a/packages/control-plane/src/sandbox/providers/modal-provider.ts +++ b/packages/control-plane/src/sandbox/providers/modal-provider.ts @@ -84,7 +84,7 @@ export interface ModalImageBuildProvider { * @example * ```typescript * const client = createModalClient(secret, workspace, environmentWebSuffix); - * const provider = new ModalSandboxProvider(client); + * const provider = new ModalSandboxProvider(client, "modal"); * * try { * const result = await provider.createSandbox(config); diff --git a/packages/modal-infra/src/sandbox/build_session.py b/packages/modal-infra/src/sandbox/build_session.py index c01c0d4b8f..121a09482f 100644 --- a/packages/modal-infra/src/sandbox/build_session.py +++ b/packages/modal-infra/src/sandbox/build_session.py @@ -3,7 +3,7 @@ import json import time from dataclasses import dataclass -from typing import cast +from typing import Any, cast import modal @@ -86,7 +86,7 @@ async def create( user_env_vars: dict[str, str] | None = None, build_execution_timeout_seconds: int = DEFAULT_BUILD_TIMEOUT_SECONDS, timeout_seconds: int = DEFAULT_BUILD_TIMEOUT_SECONDS, - sandbox_settings: dict | None = None, + sandbox_settings: dict[str, Any] | None = None, sandbox_backend: ModalBackend = "modal", ) -> BuildSessionLaunch: start_time = time.time() diff --git a/packages/modal-infra/src/web_api.py b/packages/modal-infra/src/web_api.py index 3c754f5a4c..6ec4dc9b4a 100644 --- a/packages/modal-infra/src/web_api.py +++ b/packages/modal-infra/src/web_api.py @@ -958,7 +958,7 @@ async def api_recover_build_sandbox( authorization: str | None = Header(None), x_trace_id: str | None = Header(None), x_request_id: str | None = Header(None), -) -> dict: +) -> dict[str, Any]: """Recover an owned build source whose create response was lost.""" async with _execute_endpoint( endpoint_name="api_recover_build_sandbox", From db10b2aa429ca6414270aedf4357d4631628d140 Mon Sep 17 00:00:00 2001 From: Cole Murray Date: Tue, 22 Sep 2026 01:06:46 -0700 Subject: [PATCH 03/22] style(terraform): trim backend contract test whitespace --- .../environments/production/tests/modal_backends.tftest.hcl | 1 - 1 file changed, 1 deletion(-) diff --git a/terraform/environments/production/tests/modal_backends.tftest.hcl b/terraform/environments/production/tests/modal_backends.tftest.hcl index 05b207fcd7..0093ecfb37 100644 --- a/terraform/environments/production/tests/modal_backends.tftest.hcl +++ b/terraform/environments/production/tests/modal_backends.tftest.hcl @@ -67,4 +67,3 @@ run "vm_requires_modal_credentials" { } expect_failures = [var.modal_token_id] } - From c24c9203825d67443c0658ce2f5dad85dca262c6 Mon Sep 17 00:00:00 2001 From: Cole Murray Date: Tue, 22 Sep 2026 10:17:02 -0700 Subject: [PATCH 04/22] fix: close Modal VM lifecycle review gaps --- .github/workflows/terraform.yml | 8 + .../2026-09-22-modal-vm-backend-identities.md | 19 ++ .../control-plane/src/image-builds/model.ts | 24 +-- packages/control-plane/src/sandbox/client.ts | 26 ++- .../src/sandbox/lifecycle/manager.ts | 82 +++++++- .../lifecycle/rejected-allocation.test.ts | 130 ++++++++++++ .../src/sandbox/lifecycle/test-helpers.ts | 10 +- .../control-plane/src/sandbox/provider.ts | 10 + .../sandbox/providers/modal-backends.test.ts | 46 +++- .../src/sandbox/providers/modal-provider.ts | 76 ++++--- .../control-plane/src/session/components.ts | 1 + .../handlers/child-summary.handler.test.ts | 1 + .../session-lifecycle.handler.test.ts | 1 + .../src/session/sandbox-repository.test.ts | 14 +- .../src/session/sandbox-repository.ts | 35 ++-- .../session/sandbox-shutdown-repository.ts | 1 + .../src/session/sandbox-shutdown.test.ts | 59 ++++++ .../src/session/sandbox-shutdown.ts | 124 ++++++++--- packages/control-plane/src/session/schema.ts | 6 + packages/control-plane/src/session/types.ts | 1 + .../src/session/websocket-manager.test.ts | 1 + .../test/integration/sandbox-shutdown.test.ts | 64 ++++++ packages/modal-infra/deploy.py | 16 ++ .../modal-infra/src/sandbox/launch_policy.py | 12 +- packages/modal-infra/src/sandbox/manager.py | 82 +++++--- .../src/sandbox/terminal_snapshot.py | 47 +++++ packages/modal-infra/src/web_api.py | 63 ++++-- packages/modal-infra/tests/test_deploy.py | 32 ++- .../modal-infra/tests/test_docker_launch.py | 14 +- .../modal-infra/tests/test_sandbox_launch.py | 55 +++-- .../tests/test_snapshot_timeout.py | 73 ++++++- .../tests/test_web_api_build_sandbox.py | 198 +++++++++++++++++- .../src/sandbox_runtime/docker_service.py | 61 +++--- .../src/sandbox_runtime/supervisor.py | 8 +- .../tests/test_docker_service.py | 11 +- .../tests/test_supervisor_lifecycle.py | 21 +- packages/shared/src/types/image-builds.ts | 13 ++ packages/web/src/lib/sandbox-provider.ts | 21 +- .../tests/modal_backends.tftest.hcl | 8 + .../environments/production/variables.tf | 4 +- terraform/modules/modal-app/outputs.tf | 5 + 41 files changed, 1237 insertions(+), 246 deletions(-) create mode 100644 packages/modal-infra/src/sandbox/terminal_snapshot.py diff --git a/.github/workflows/terraform.yml b/.github/workflows/terraform.yml index b6a5024863..71f72cc959 100644 --- a/.github/workflows/terraform.yml +++ b/.github/workflows/terraform.yml @@ -120,6 +120,13 @@ jobs: run: terraform test working-directory: ${{ env.TF_WORKING_DIR }} + - name: Modal module contract tests + id: modal_test + run: | + terraform init -backend=false + terraform test + working-directory: terraform/modules/modal-app + - name: Post Validation Results if: always() && github.event_name == 'pull_request' uses: actions/github-script@v8 @@ -138,6 +145,7 @@ jobs: | Init | ${{ steps.init.outcome == 'success' && '✅' || '❌' }} | | Validate | ${{ steps.validate.outcome == 'success' && '✅' || '❌' }} | | Tests | ${{ steps.test.outcome == 'success' && '✅' || '❌' }} | + | Modal module tests | ${{ steps.modal_test.outcome == 'success' && '✅' || '❌' }} | ${planNote} *Pushed by: @${{ github.actor }}, Action: \`${{ github.event_name }}\`*`; diff --git a/docs/plans/2026-09-22-modal-vm-backend-identities.md b/docs/plans/2026-09-22-modal-vm-backend-identities.md index d4a271f668..cb8ffc1a54 100644 --- a/docs/plans/2026-09-22-modal-vm-backend-identities.md +++ b/docs/plans/2026-09-22-modal-vm-backend-identities.md @@ -345,6 +345,25 @@ Docker over a local control socket, the provider captures the filesystem, then w retirement. Only then does the response confirm `sourceStopped`; subsequent work restores into a new generation. Standard Modal snapshots are unchanged. This avoids a new general snapshot protocol. +Review hardening preserves those boundaries: + +- VM session generations share a provider-enforced allocation name, with exact generation ownership + tags. A pending provider reference is stored before create/restore so snapshot and stop can + resolve an allocation whose HTTP response was lost; this reference is not startup confirmation. +- A terminal VM capture uses that generation's stable source reference as its operation key. Modal + stores its image/source receipt before retirement, and a retry can retrieve it without the source + still existing. After timeout or restart, the lifecycle uses a dedicated read-only receipt lookup, + commits any recovered image, then confirms retirement using the recorded immutable source ID. No + unknown capture is reissued. Modal Dict receipts expire after seven days without access. An + incomplete capture intent stays unknown; this does not guarantee recovery if the underlying + snapshot SDK response is itself lost before the receipt is recorded. +- Rejected allocations are durably fenced before awaited cleanup. Their explicit cleanup marker + rearms retirement retries after restart without changing unrelated snapshot/recovery holds. +- Switching back to gVisor carries forward the currently deployed verified VM image through a + private deployment handshake. It does not build another VM image or strip capability before the + worker selector cutover. Removing that retained capability is a separate operator action, not an + automatic side effect of switching compute offerings. + Validate these areas: - **Checkpoint preparation:** current ordinary provider capture calls filesystem snapshot directly. diff --git a/packages/control-plane/src/image-builds/model.ts b/packages/control-plane/src/image-builds/model.ts index e3cd1b9a31..246cae2482 100644 --- a/packages/control-plane/src/image-builds/model.ts +++ b/packages/control-plane/src/image-builds/model.ts @@ -13,9 +13,10 @@ import { formatRepositoryFullName, parseRepositoryFullName, } from "@open-inspect/shared/types/repositories"; -import type { - ImageBuildScopeKind, - ImageBuildStatus, +import { + IMAGE_BUILD_PROVIDER_IDS, + type ImageBuildScopeKind, + type ImageBuildStatus, } from "@open-inspect/shared/types/image-builds"; import type { HarnessId } from "@open-inspect/shared/harnesses"; import { z } from "zod"; @@ -24,22 +25,7 @@ import { MIN_COMPATIBLE_RUNTIME_GENERATION, } from "../sandbox/runtime-manifest"; -/** - * Providers with image-build support: Modal images, Vercel snapshots, - * OpenComputer checkpoints, E2B snapshots, Daytona snapshots. - * - * Support is the provider's ability to build and boot an artifact. Whether a - * deployment may start new Daytona builds is a separate, operator-owned - * question — see `resolveImageBuildAdmission` in provider-policy.ts. - */ -export const IMAGE_BUILD_PROVIDER_IDS = [ - "modal", - "modal-vm", - "vercel", - "opencomputer", - "e2b", - "daytona", -] as const; +export { IMAGE_BUILD_PROVIDER_IDS } from "@open-inspect/shared/types/image-builds"; export const imageBuildProviderSchema = z.enum(IMAGE_BUILD_PROVIDER_IDS); diff --git a/packages/control-plane/src/sandbox/client.ts b/packages/control-plane/src/sandbox/client.ts index be658c23e6..e256c353b0 100644 --- a/packages/control-plane/src/sandbox/client.ts +++ b/packages/control-plane/src/sandbox/client.ts @@ -233,6 +233,7 @@ export interface RestoreSandboxResponse { export interface SnapshotSandboxRequest { providerObjectId: string; sessionId: string; + sandboxBackend?: ModalBackend; signal?: AbortSignal; deadlineAtMs?: number; } @@ -319,6 +320,7 @@ export class ModalApiError extends Error { export class ModalClient { private createSandboxUrl: string; private snapshotSandboxUrl: string; + private recoverSandboxSnapshotUrl: string; private snapshotBuildSandboxUrl: string; private restoreSandboxUrl: string; private stopSandboxUrl: string; @@ -367,6 +369,7 @@ export class ModalClient { modalEndpointUrl(functionName, workspace, environmentWebSuffix, apiUrl); this.createSandboxUrl = url("api-create-sandbox"); this.snapshotSandboxUrl = url("api-snapshot-sandbox"); + this.recoverSandboxSnapshotUrl = url("api-recover-sandbox-snapshot"); this.snapshotBuildSandboxUrl = url("api-snapshot-build-sandbox"); this.restoreSandboxUrl = url("api-restore-sandbox"); this.stopSandboxUrl = url("api-stop-sandbox"); @@ -542,8 +545,28 @@ export class ModalClient { } /** - * Trigger a filesystem snapshot for a sandbox object. + * Read a previously recorded terminal snapshot receipt without starting a capture. */ + async recoverSandboxSnapshot( + request: SnapshotSandboxRequest + ): Promise<{ imageId: string } | null> { + const result = await this.postJson( + this.recoverSandboxSnapshotUrl, + "recoverSandboxSnapshot", + MODAL_CLEANUP_REQUEST_DEADLINE_MS, + { sandbox_id: request.providerObjectId }, + z.object({ + success: z.literal(true), + data: z.object({ image_id: z.string().min(1).nullable() }), + }), + undefined, + request.signal, + () => {} + ); + return result.data.image_id ? { imageId: result.data.image_id } : null; + } + + /** Trigger a filesystem snapshot for a sandbox object. */ async snapshotSandbox( request: SnapshotSandboxRequest, correlation?: CorrelationContext @@ -566,6 +589,7 @@ export class ModalClient { { sandbox_id: request.providerObjectId, deadline_at_ms: request.deadlineAtMs ?? null, + ...(request.sandboxBackend ? { sandbox_backend: request.sandboxBackend } : {}), }, snapshotSandboxModalResponseSchema, correlation, diff --git a/packages/control-plane/src/sandbox/lifecycle/manager.ts b/packages/control-plane/src/sandbox/lifecycle/manager.ts index 4e9472c12d..7fcebe8c8e 100644 --- a/packages/control-plane/src/sandbox/lifecycle/manager.ts +++ b/packages/control-plane/src/sandbox/lifecycle/manager.ts @@ -96,6 +96,7 @@ const log = createLogger("lifecycle-manager"); /** TTL for terminal auth JWTs (24 hours, matching typical sandbox lifetime). */ const TERMINAL_TOKEN_TTL_SECONDS = 86400; const PROVIDER_REPLACEMENT_STOP_TIMEOUT_MS = 10_000; +const REJECTED_ALLOCATION_CLEANUP_RETRY_MS = 30_000; // ==================== Dependency Interfaces ==================== @@ -210,12 +211,19 @@ export interface SandboxStorage { from: SandboxStatus, to: SandboxStatus ): boolean; + /** + * Fence a rejected generation and retain its cleanup handle without replacing + * a terminal status. Superseded generations cannot modify the current row. + */ + rejectProviderStartup( + generation: SandboxGeneration, + providerObjectId: string | null + ): "failed" | "retained" | "superseded"; /** * Atomically accept a provider startup result for the named generation, * store its handle, and advance a fresh spawn to connecting. Returns the * resulting status, or null when another lifecycle event owns the row. */ - rejectProviderStartup(generation: SandboxGeneration, providerObjectId: string | null): boolean; commitProviderStartup( generation: SandboxGeneration, providerObjectId: string | null, @@ -813,6 +821,7 @@ export class SandboxLifecycleManager let result: CreateSandboxResult; try { + this.recordPendingProviderReference(generation, sessionId); result = await this.provider.createSandbox(createConfig); } catch (error) { if (!selectedImage) throw error; @@ -848,6 +857,7 @@ export class SandboxLifecycleManager preserveProviderObjectId: false, shutdownPolicy: shutdownPolicyForLaunch("new", null), })); + this.recordPendingProviderReference(generation, sessionId); result = await this.provider.createSandbox({ ...createConfig, sandboxId: expectedSandboxId, @@ -1196,6 +1206,7 @@ export class SandboxLifecycleManager const sandboxSettings = this.parseSandboxSettings(session); const timeoutSeconds = this.resolveSandboxTimeoutSeconds(sandboxSettings); this.shutdown.markRecoveryInvoked(generation); + this.recordPendingProviderReference(generation, session.session_name || session.id); const result = await this.provider.restoreFromSnapshot({ snapshotImageId, retireSandboxId: priorSandboxId, @@ -2080,7 +2091,15 @@ export class SandboxLifecycleManager } } - handleShutdownAlarm(): Promise<"continue" | "hold_watchdogs"> { + async handleShutdownAlarm(): Promise<"continue" | "hold_watchdogs"> { + const rejected = this.storage.getSandbox(); + if (rejected?.startup_rejected && rejected.modal_object_id) { + await this.cleanupRejectedAllocation( + { sandboxId: rejected.modal_sandbox_id, createdAt: rejected.created_at }, + rejected.modal_object_id + ); + return "hold_watchdogs"; + } return this.shutdown.handleAlarm(); } @@ -2230,23 +2249,66 @@ export class SandboxLifecycleManager await this.storage.updateSandboxAccess("ttyd", url, token); } + private recordPendingProviderReference(generation: SandboxGeneration, sessionId: string): void { + if (!generation.sandboxId) throw new SpawnSupersededError(); + const reference = this.provider.pendingSandboxReference?.(sessionId, generation.sandboxId); + if (!reference) return; + const row = this.storage.getSandbox(); + if ( + row?.modal_sandbox_id !== generation.sandboxId || + row.created_at !== generation.createdAt || + row.fenced + ) { + throw new SpawnSupersededError(); + } + this.storage.updateSandboxModalObjectId(reference); + } + private async retainRejectedAllocation( error: unknown, generation: SandboxGeneration | null ): Promise { if (!(error instanceof SandboxLaunchRejectedError) || !generation) return; - const alreadyFailed = this.storage.getSandbox()?.status === "failed"; // Keep the rejected handle on this generation so restart/replacement retries retirement. // It is not a successful startup and receives no access credentials or readiness signal. - if (!this.storage.rejectProviderStartup(generation, error.providerObjectId)) { + const rejection = this.storage.rejectProviderStartup(generation, error.providerObjectId); + if (rejection === "superseded") { await this.destroyLateProviderResult(error.providerObjectId ?? undefined); return; } this.wsManager.detachSandboxWebSocket(1008, "Provider allocation rejected"); this.clearSandboxAccessState(); - this.broadcaster.broadcast({ type: "sandbox_status", status: "failed" }); - this.reportSandboxError(error.message); - if (!alreadyFailed) this.recordSpawnFailure(Date.now(), generation.createdAt); + if (rejection === "failed") { + this.broadcaster.broadcast({ type: "sandbox_status", status: "failed" }); + this.reportSandboxError(error.message); + this.recordSpawnFailure(Date.now(), generation.createdAt); + } + if (error.providerObjectId) + await this.cleanupRejectedAllocation(generation, error.providerObjectId); + } + + async rearmRejectedAllocationCleanup(): Promise { + const row = this.storage.getSandbox(); + if (row?.startup_rejected && row.modal_object_id) { + await this.alarmScheduler.schedule(Date.now() + REJECTED_ALLOCATION_CLEANUP_RETRY_MS); + } + } + + private async cleanupRejectedAllocation( + generation: SandboxGeneration, + providerObjectId: string + ): Promise { + // Persist the next attempt before provider I/O so an eviction cannot lose cleanup. + await this.rearmRejectedAllocationCleanup(); + if (!(await this.destroyLateProviderResult(providerObjectId))) return; + const row = this.storage.getSandbox(); + if ( + row?.modal_sandbox_id === generation.sandboxId && + row.created_at === generation.createdAt && + row.modal_object_id === providerObjectId + ) { + this.storage.updateSandboxModalObjectId(null); + } } private async claimProviderStartup( @@ -2293,8 +2355,8 @@ export class SandboxLifecycleManager return true; } - private async destroyLateProviderResult(providerObjectId: string | undefined): Promise { - if (!providerObjectId || !this.canStopProviderSandbox()) return; + private async destroyLateProviderResult(providerObjectId: string | undefined): Promise { + if (!providerObjectId || !this.canStopProviderSandbox()) return false; const controller = new AbortController(); let timeoutId: ReturnType | undefined; try { @@ -2313,11 +2375,13 @@ export class SandboxLifecycleManager ), timeout, ]); + return true; } catch (error) { this.log.warn("Failed to destroy superseded provider sandbox", { provider_object_id: providerObjectId, error: error instanceof Error ? error.message : String(error), }); + return false; } finally { if (timeoutId !== undefined) clearTimeout(timeoutId); } diff --git a/packages/control-plane/src/sandbox/lifecycle/rejected-allocation.test.ts b/packages/control-plane/src/sandbox/lifecycle/rejected-allocation.test.ts index f7a2c29b2f..5c7e71dcc3 100644 --- a/packages/control-plane/src/sandbox/lifecycle/rejected-allocation.test.ts +++ b/packages/control-plane/src/sandbox/lifecycle/rejected-allocation.test.ts @@ -1,8 +1,94 @@ import { describe, expect, it, vi } from "vitest"; +import { COMPATIBLE_RUNTIME_VERSION } from "../../image-builds/test-helpers"; +import { createAlarmHandler } from "../../session/alarm/handler"; import { SandboxLaunchRejectedError } from "../provider"; +import { ModalSandboxProvider } from "../providers/modal-provider"; +import type { ModalClient } from "../client"; import { createAlarmFixture, createMockProvider, createMockSandbox } from "./test-helpers"; describe("rejected provider allocation", () => { + it.each(["stopped", "stale"] as const)( + "preserves %s when a cancelled launch is later rejected", + async (status) => { + const sandbox = createMockSandbox({ status: "pending", modal_object_id: null }); + const provider = createMockProvider({ + capabilities: { supportsExplicitStop: true }, + stopSandbox: async () => { + throw new Error("unavailable"); + }, + createSandbox: async () => { + sandbox.status = status; + throw new SandboxLaunchRejectedError("incompatible", "sb-rejected"); + }, + }); + const fixture = createAlarmFixture(sandbox, provider); + await fixture.manager.spawnSandbox(); + expect(sandbox.status).toBe(status); + expect(sandbox.fenced).toBe(1); + expect(sandbox.modal_object_id).toBe("sb-rejected"); + expect(fixture.broadcaster.broadcast).not.toHaveBeenCalledWith({ + type: "sandbox_status", + status: "failed", + }); + expect(fixture.storage.incrementCircuitBreakerFailure).not.toHaveBeenCalled(); + } + ); + it("retains a pre-launch VM reference when the response is lost after bridge connection", async () => { + const sandbox = createMockSandbox({ status: "pending", modal_object_id: null }); + const client = { + createSandbox: vi.fn(async () => { + expect(sandbox.modal_object_id).toContain("modal-vm-session:"); + sandbox.status = "ready"; + vi.mocked(fixture.wsManager.getSandboxWebSocket).mockReturnValue({} as WebSocket); + throw new Error("response lost"); + }), + }; + const provider = new ModalSandboxProvider(client as unknown as ModalClient, "modal-vm"); + const fixture = createAlarmFixture(sandbox, provider); + await fixture.manager.spawnSandbox(); + expect(sandbox.status).toBe("ready"); + expect(sandbox.modal_object_id).toContain(sandbox.modal_sandbox_id); + const restarted = createAlarmFixture(sandbox, provider); + expect(restarted.storage.getSandbox()?.modal_object_id).toBe(sandbox.modal_object_id); + }); + it("fences an early connected generation before waiting for mismatch cleanup", async () => { + const sandbox = createMockSandbox({ status: "pending", modal_object_id: null }); + let finishStop!: () => void; + const client = { + createSandbox: vi.fn(async () => { + sandbox.status = "ready"; + vi.mocked(fixture.wsManager.getSandboxWebSocket).mockReturnValue({} as WebSocket); + return { + sandboxId: sandbox.modal_sandbox_id, + modalObjectId: "sb-rejected", + sandboxBackend: "modal", + createdAt: 1, + }; + }), + stopSandbox: vi.fn( + () => + new Promise((resolve) => { + finishStop = resolve; + }) + ), + }; + const fixture = createAlarmFixture( + sandbox, + new ModalSandboxProvider(client as unknown as ModalClient, "modal-vm") + ); + const spawning = fixture.manager.spawnSandbox(); + try { + await vi.waitFor(() => expect(client.stopSandbox).toHaveBeenCalledOnce()); + expect(sandbox.fenced).toBe(1); + expect(sandbox.auth_token_hash).toBe(""); + expect(sandbox.modal_object_id).toBe("sb-rejected"); + expect(fixture.wsManager.detachSandboxWebSocket).toHaveBeenCalled(); + expect(fixture.manager.mayProcessQueuedWork()).toBe(false); + } finally { + finishStop(); + await spawning; + } + }); it.each([ [false, "sb-rejected"], [true, "sb-rejected"], @@ -16,6 +102,9 @@ describe("rejected provider allocation", () => { const sandbox = createMockSandbox({ status: "pending", modal_object_id: null }); const provider = createMockProvider({ capabilities: { supportsExplicitStop: true }, + stopSandbox: async () => { + throw new Error("retirement unavailable"); + }, createSandbox: async () => { if (earlyBridge) { sandbox.status = earlyBridge === "expired" ? "failed" : "connecting"; @@ -55,4 +144,45 @@ describe("rejected provider allocation", () => { expect(sandbox.fenced).toBe(1); expect(provider.createSandbox).toHaveBeenCalledTimes(1); }); + it.each(["create", "restore"] as const)( + "rearms %s cleanup through the assembled alarm handler even under a shutdown hold", + async (launch) => { + const sandbox = createMockSandbox({ status: "pending", modal_object_id: null }); + const stopSandbox = vi.fn().mockRejectedValue(new Error("provider unavailable")); + const provider = createMockProvider({ + capabilities: { supportsExplicitStop: true }, + stopSandbox, + createSandbox: async () => { + throw new SandboxLaunchRejectedError("incompatible", "sb-rejected"); + }, + }); + const first = createAlarmFixture(sandbox, provider); + if (launch === "restore") { + vi.mocked(first.shutdown.startupDecision).mockReturnValue({ + kind: "restore_snapshot", + snapshotId: "im-existing", + runtimeVersion: COMPATIBLE_RUNTIME_VERSION, + }); + provider.restoreFromSnapshot = provider.createSandbox as never; + } + await first.manager.spawnSandbox(); + const restarted = createAlarmFixture(sandbox, provider); + vi.spyOn(restarted.shutdown, "handleAlarm").mockImplementation( + async () => "hold_watchdogs" as never + ); + const handler = createAlarmHandler({ + preserveBeforeWatchdogs: () => restarted.manager.handleShutdownAlarm(), + lifecycleManager: restarted.manager, + terminalMessageProjection: { flushPending: vi.fn(async () => {}) }, + } as never); + await restarted.manager.rearmRejectedAllocationCleanup(); + expect(restarted.alarmScheduler.schedule).toHaveBeenCalled(); + await handler.handle(); + expect(sandbox.modal_object_id).toBe("sb-rejected"); + stopSandbox.mockResolvedValue({ success: true }); + await handler.handle(); + expect(sandbox.modal_object_id).toBeNull(); + expect(sandbox.fenced).toBe(1); + } + ); }); diff --git a/packages/control-plane/src/sandbox/lifecycle/test-helpers.ts b/packages/control-plane/src/sandbox/lifecycle/test-helpers.ts index dc9ef32dc8..31292dcdc3 100644 --- a/packages/control-plane/src/sandbox/lifecycle/test-helpers.ts +++ b/packages/control-plane/src/sandbox/lifecycle/test-helpers.ts @@ -97,6 +97,7 @@ export function createMockSandbox( boot_phase: null, boot_seq: null, fenced: 0, + startup_rejected: 0, created_at: Date.now() - 60000, spawn_failure_count: 0, last_spawn_failure: 0, @@ -168,14 +169,16 @@ export function createMockStorage( sandbox.modal_sandbox_id !== generation.sandboxId || sandbox.created_at !== generation.createdAt ) - return false; + return "superseded"; sandbox.modal_object_id = providerObjectId; - sandbox.status = "failed"; + const failed = ["spawning", "connecting", "ready"].includes(sandbox.status); + if (failed) sandbox.status = "failed"; sandbox.fenced = 1; + sandbox.startup_rejected = 1; sandbox.auth_token_hash = ""; sandbox.auth_token = null; sandbox.active_socket_id = ""; - return true; + return failed ? "failed" : "retained"; }), commitProviderStartup: vi.fn((generation, providerObjectId, allowFailedSelfHeal) => { calls.push("commitProviderStartup"); @@ -199,6 +202,7 @@ export function createMockStorage( calls.push("updateSandboxForSpawn"); if (sandbox) { sandbox.status = data.status; + sandbox.startup_rejected = 0; sandbox.created_at = data.createdAt; sandbox.auth_token_hash = ""; sandbox.auth_token = null; diff --git a/packages/control-plane/src/sandbox/provider.ts b/packages/control-plane/src/sandbox/provider.ts index 3baaccf0e0..1916fccf8c 100644 --- a/packages/control-plane/src/sandbox/provider.ts +++ b/packages/control-plane/src/sandbox/provider.ts @@ -564,6 +564,11 @@ export interface SandboxProvider { /** Provider capabilities */ readonly capabilities: SandboxProviderCapabilities; + /** Optional opaque reference usable for snapshot/stop even if the launch response is lost. + * Persisted before launch; this is not evidence that startup succeeded. + */ + pendingSandboxReference?(sessionId: string, sandboxId: string): string | undefined; + /** * Create a new sandbox. * @@ -602,6 +607,11 @@ export interface SandboxProvider { */ takeSnapshot?(config: SnapshotConfig): Promise; + /** Recover only an existing terminal capture receipt. Must never initiate a snapshot or stop. */ + recoverSnapshotReceipt?( + config: Pick + ): Promise<{ imageId: string } | null>; + /** * Stop a sandbox explicitly via the provider API. * diff --git a/packages/control-plane/src/sandbox/providers/modal-backends.test.ts b/packages/control-plane/src/sandbox/providers/modal-backends.test.ts index dfccdd876c..8c004e3b4c 100644 --- a/packages/control-plane/src/sandbox/providers/modal-backends.test.ts +++ b/packages/control-plane/src/sandbox/providers/modal-backends.test.ts @@ -55,6 +55,46 @@ function fixture(confirmation: unknown) { afterEach(() => vi.unstubAllGlobals()); describe("distinct Modal backend identities", () => { + it("uses a dedicated read-only receipt endpoint after the original capture deadline", async () => { + const fetchMock = vi.fn().mockResolvedValue( + new Response( + JSON.stringify({ + success: true, + data: { image_id: "im-recovered" }, + }), + { status: 200 } + ) + ); + vi.stubGlobal("fetch", fetchMock); + const provider = new ModalSandboxProvider(createModalClient("secret", "acme"), "modal-vm"); + await expect( + provider.recoverSnapshotReceipt({ + providerObjectId: 'modal-vm-session:["session","generation"]', + sessionId: "session", + deadlineAtMs: Date.now() + 30_000, + }) + ).resolves.toEqual({ imageId: "im-recovered" }); + const [url, options] = fetchMock.mock.calls[0]; + expect(url).toContain("api-recover-sandbox-snapshot"); + expect(JSON.parse(options.body)).toEqual({ + sandbox_id: 'modal-vm-session:["session","generation"]', + }); + expect(fetchMock).toHaveBeenCalledOnce(); + }); + it("recovers a lost terminal snapshot response using the same source reference and deadline", async () => { + const { client, provider } = fixture("modal-vm"); + client.snapshotSandbox.mockRejectedValueOnce(new Error("response lost")); + await expect( + provider.takeSnapshot({ + providerObjectId: "sb-1", + sessionId: "session-1", + reason: "shutdown", + deadlineAtMs: Date.now() + 60_000, + }) + ).resolves.toMatchObject({ success: true, imageId: "im-1", sourceStopped: true }); + expect(client.snapshotSandbox).toHaveBeenCalledTimes(2); + expect(client.snapshotSandbox.mock.calls[0]).toEqual(client.snapshotSandbox.mock.calls[1]); + }); it("selects the immutable backend on both launch paths without altering generic resources", async () => { const { provider, client } = fixture("modal-vm"); const settings = { cpuCores: 3, memoryMib: null }; @@ -79,18 +119,18 @@ describe("distinct Modal backend identities", () => { }); it.each([undefined, null, false, "modal", "future-backend", { unexpected: true }])( - "rejects and retires create/restore allocations with confirmation %j", + "returns rejected create/restore handles before cleanup with confirmation %j", async (value) => { const { provider, client } = fixture(value); await expect(provider.createSandbox(config)).rejects.toThrow("did not confirm"); await expect( provider.restoreFromSnapshot({ ...config, snapshotImageId: "im-1" }) ).rejects.toThrow("did not confirm"); - expect(client.stopSandbox).toHaveBeenCalledTimes(2); + expect(client.stopSandbox).not.toHaveBeenCalled(); } ); - it("retains a rejected session allocation ID when retirement fails", async () => { + it("carries the rejected session allocation ID for lifecycle-owned cleanup", async () => { const { provider, client } = fixture("modal"); client.stopSandbox.mockRejectedValue(new Error("unreachable")); await expect(provider.createSandbox(config)).rejects.toMatchObject({ diff --git a/packages/control-plane/src/sandbox/providers/modal-provider.ts b/packages/control-plane/src/sandbox/providers/modal-provider.ts index cb6b9384db..a5514471c1 100644 --- a/packages/control-plane/src/sandbox/providers/modal-provider.ts +++ b/packages/control-plane/src/sandbox/providers/modal-provider.ts @@ -100,6 +100,12 @@ export class ModalSandboxProvider implements SandboxProvider, ModalImageBuildPro readonly capabilities: SandboxProviderCapabilities; + pendingSandboxReference(sessionId: string, sandboxId: string): string | undefined { + return this.name === "modal-vm" + ? `modal-vm-session:${JSON.stringify([sessionId, sandboxId])}` + : undefined; + } + constructor( private readonly client: ModalClient, backend: ModalBackend @@ -151,7 +157,7 @@ export class ModalSandboxProvider implements SandboxProvider, ModalImageBuildPro config.correlation ); - await this.confirmSessionLaunch(config, result); + this.confirmSessionLaunch(result); return { sandboxId: result.sandboxId, providerObjectId: result.modalObjectId, @@ -210,7 +216,7 @@ export class ModalSandboxProvider implements SandboxProvider, ModalImageBuildPro config.correlation ); - await this.confirmSessionLaunch(config, result); + this.confirmSessionLaunch(result); return { success: true, sandboxId: result.sandboxId, @@ -246,15 +252,27 @@ export class ModalSandboxProvider implements SandboxProvider, ModalImageBuildPro */ async takeSnapshot(config: SnapshotConfig): Promise { try { - const result = await this.client.snapshotSandbox( - { - providerObjectId: config.providerObjectId, - sessionId: config.sessionId, - signal: signalUntilDeadline(config.deadlineAtMs, config.signal), - deadlineAtMs: config.deadlineAtMs, - }, - config.correlation - ); + const request = { + providerObjectId: config.providerObjectId, + sessionId: config.sessionId, + sandboxBackend: this.name, + signal: signalUntilDeadline(config.deadlineAtMs, config.signal), + deadlineAtMs: config.deadlineAtMs, + }; + let result; + try { + result = await this.client.snapshotSandbox(request, config.correlation); + } catch (error) { + // A VM's terminal capture is keyed by this stable source reference. + // One retry can recover the durable receipt after a lost response. + if ( + this.name !== "modal-vm" || + request.signal?.aborted || + (error instanceof ModalApiError && error.status < 500) + ) + throw error; + result = await this.client.snapshotSandbox(request, config.correlation); + } if (this.name === "modal-vm" && result.sourceStopped !== true) { throw new SandboxProviderError( @@ -282,6 +300,16 @@ export class ModalSandboxProvider implements SandboxProvider, ModalImageBuildPro } } + recoverSnapshotReceipt( + config: Pick + ): Promise<{ imageId: string } | null> { + if (this.name !== "modal-vm") return Promise.resolve(null); + return this.client.recoverSandboxSnapshot({ + ...config, + signal: signalUntilDeadline(config.deadlineAtMs, config.signal), + }); + } + async stopSandbox(config: StopConfig): Promise { try { const signal = signalUntilDeadline(config.deadlineAtMs, config.signal); @@ -372,30 +400,18 @@ export class ModalSandboxProvider implements SandboxProvider, ModalImageBuildPro ); } - private async confirmSessionLaunch( - config: CreateSandboxConfig | RestoreConfig, - result: { modalObjectId?: string; sandboxBackend?: unknown; legacyDockerEnabled?: unknown } - ): Promise { + private confirmSessionLaunch(result: { + modalObjectId?: string; + sandboxBackend?: unknown; + legacyDockerEnabled?: unknown; + }): void { try { this.assertBackend(result); } catch (error) { - if (result.modalObjectId) { - try { - await this.client.stopSandbox( - { providerObjectId: result.modalObjectId, sessionId: config.sessionId }, - config.correlation - ); - } catch (cleanupError) { - throw new SandboxLaunchRejectedError( - "Incompatible Modal allocation could not be retired", - result.modalObjectId, - cleanupError instanceof Error ? cleanupError : undefined - ); - } - } + // The lifecycle must persist and fence this generation before any cleanup await. throw new SandboxLaunchRejectedError( error instanceof Error ? error.message : "Incompatible Modal allocation", - null, + result.modalObjectId ?? null, error instanceof Error ? error : undefined ); } diff --git a/packages/control-plane/src/session/components.ts b/packages/control-plane/src/session/components.ts index 32e51a15f4..0669e25a17 100644 --- a/packages/control-plane/src/session/components.ts +++ b/packages/control-plane/src/session/components.ts @@ -974,6 +974,7 @@ export function createSessionRuntime(platform: SessionPlatform, env: Env): Sessi async () => { await wsManager.expireAuthorizationLeases(Date.now()); await alarmScheduler.rehydrate(); + await lifecycleManager.rearmRejectedAllocationCleanup(); await terminalMessageProjection.rearm(); }, { diff --git a/packages/control-plane/src/session/http/handlers/child-summary.handler.test.ts b/packages/control-plane/src/session/http/handlers/child-summary.handler.test.ts index b72a34422e..d6240b3f4b 100644 --- a/packages/control-plane/src/session/http/handlers/child-summary.handler.test.ts +++ b/packages/control-plane/src/session/http/handlers/child-summary.handler.test.ts @@ -75,6 +75,7 @@ function createSandbox(overrides: Partial = {}): SandboxRow { boot_phase: null, boot_seq: null, fenced: 0, + startup_rejected: 0, created_at: 1, ...overrides, }; diff --git a/packages/control-plane/src/session/http/handlers/session-lifecycle.handler.test.ts b/packages/control-plane/src/session/http/handlers/session-lifecycle.handler.test.ts index 903916b6f7..2152b41bf4 100644 --- a/packages/control-plane/src/session/http/handlers/session-lifecycle.handler.test.ts +++ b/packages/control-plane/src/session/http/handlers/session-lifecycle.handler.test.ts @@ -69,6 +69,7 @@ function createSandbox(overrides: Partial = {}): SandboxRow { boot_phase: null, boot_seq: null, fenced: 0, + startup_rejected: 0, created_at: 1, ...overrides, }; diff --git a/packages/control-plane/src/session/sandbox-repository.test.ts b/packages/control-plane/src/session/sandbox-repository.test.ts index a492ab4f0d..5e8ab9193c 100644 --- a/packages/control-plane/src/session/sandbox-repository.test.ts +++ b/packages/control-plane/src/session/sandbox-repository.test.ts @@ -36,6 +36,7 @@ function sandboxRow(overrides: Partial = {}): SandboxRow { boot_phase: null, boot_seq: null, fenced: 0, + startup_rejected: 0, created_at: 1000, ...overrides, }; @@ -465,7 +466,7 @@ describe("SandboxRepository boot state (SQLite)", () => { describe("rejectProviderStartup", () => { const generation = { sandboxId: "sb-1", createdAt: 1000 }; - it.each(["spawning", "connecting", "ready", "failed"])( + it.each(["spawning", "connecting", "ready", "failed", "stopped", "stale"])( "fences %s and persists cleanup responsibility", (status) => { const { repository, set } = createSqliteRepository(); @@ -474,9 +475,12 @@ describe("SandboxRepository boot state (SQLite)", () => { status, status === "failed" ? 1 : 0 ); - expect(repository.rejectProviderStartup(generation, "sb-rejected")).toBe(true); + expect(repository.rejectProviderStartup(generation, "sb-rejected")).toBe( + ["spawning", "connecting", "ready"].includes(status) ? "failed" : "retained" + ); expect(repository.getSandbox()).toMatchObject({ - status: "failed", + status: ["stopped", "stale"].includes(status) ? status : "failed", + startup_rejected: 1, fenced: 1, modal_object_id: "sb-rejected", auth_token_hash: "", @@ -490,10 +494,10 @@ describe("SandboxRepository boot state (SQLite)", () => { it("records confirmed cleanup without retaining an obligation and rejects superseded writes", () => { const { repository, set } = createSqliteRepository(); set("status = 'connecting', modal_sandbox_id = 'sb-1', modal_object_id = 'old'"); - expect(repository.rejectProviderStartup(generation, null)).toBe(true); + expect(repository.rejectProviderStartup(generation, null)).toBe("failed"); expect(repository.getSandbox()?.modal_object_id).toBeNull(); expect(repository.rejectProviderStartup({ ...generation, createdAt: 999 }, "late")).toBe( - false + "superseded" ); expect(repository.getSandbox()?.modal_object_id).toBeNull(); }); diff --git a/packages/control-plane/src/session/sandbox-repository.ts b/packages/control-plane/src/session/sandbox-repository.ts index 3f0af2588d..2a9f4c7789 100644 --- a/packages/control-plane/src/session/sandbox-repository.ts +++ b/packages/control-plane/src/session/sandbox-repository.ts @@ -162,18 +162,25 @@ export class SandboxRepository { rejectProviderStartup( generation: { sandboxId: string | null; createdAt: number }, providerObjectId: string | null - ): boolean { - const result = this.sql.exec( - `UPDATE sandbox SET modal_object_id = ?, status = 'failed', fenced = 1, - auth_token_hash = '', auth_token = NULL, active_socket_id = '' - WHERE id = (SELECT id FROM sandbox LIMIT 1) - AND modal_sandbox_id IS ? AND created_at = ?`, - providerObjectId, - generation.sandboxId, - generation.createdAt - ); - result.toArray(); - return (result.rowsWritten ?? 0) > 0; + ): "failed" | "retained" | "superseded" { + const assignments = `modal_object_id = ?, fenced = 1, startup_rejected = 1, + auth_token_hash = '', auth_token = NULL, active_socket_id = ''`; + const identity = `id = (SELECT id FROM sandbox LIMIT 1) + AND modal_sandbox_id IS ? AND created_at = ?`; + const args = [providerObjectId, generation.sandboxId, generation.createdAt]; + const failed = this.sql + .exec( + `UPDATE sandbox SET ${assignments}, status = 'failed' + WHERE ${identity} AND status IN ('spawning', 'connecting', 'ready') + RETURNING id`, + ...args + ) + .toArray(); + if (failed.length) return "failed"; + const retained = this.sql + .exec(`UPDATE sandbox SET ${assignments} WHERE ${identity} RETURNING id`, ...args) + .toArray(); + return retained.length ? "retained" : "superseded"; } commitProviderStartup( @@ -293,7 +300,7 @@ export class SandboxRepository { active_socket_id = '', boot_phase = NULL, boot_seq = NULL, - fenced = 0 + fenced = 0, startup_rejected = 0 WHERE id = (SELECT id FROM sandbox LIMIT 1)`, data.status, data.createdAt, @@ -348,7 +355,7 @@ export class SandboxRepository { last_heartbeat = NULL, boot_phase = NULL, boot_seq = NULL, - fenced = 0 + fenced = 0, startup_rejected = 0 WHERE id = (SELECT id FROM sandbox LIMIT 1)`, data.status, data.createdAt diff --git a/packages/control-plane/src/session/sandbox-shutdown-repository.ts b/packages/control-plane/src/session/sandbox-shutdown-repository.ts index f708542e27..5594feea2a 100644 --- a/packages/control-plane/src/session/sandbox-shutdown-repository.ts +++ b/packages/control-plane/src/session/sandbox-shutdown-repository.ts @@ -27,6 +27,7 @@ const stateSchema = sandboxShutdownSchema lifecyclePolicy: z.enum(["confirmed", "legacy"]).optional(), restoreInvoked: z.boolean().optional(), checkpointInFlight: z.boolean().optional(), + captureReceiptPending: z.boolean().optional(), operationId: z.string().optional(), messageId: z.string().optional(), stopByMs: z.number().optional(), diff --git a/packages/control-plane/src/session/sandbox-shutdown.test.ts b/packages/control-plane/src/session/sandbox-shutdown.test.ts index 4e246db51f..7bdf4e9a73 100644 --- a/packages/control-plane/src/session/sandbox-shutdown.test.ts +++ b/packages/control-plane/src/session/sandbox-shutdown.test.ts @@ -158,6 +158,65 @@ function preparedEvent( describe("SandboxShutdownCoordinator", () => { beforeEach(() => vi.restoreAllMocks()); + it("recovers a terminal capture receipt after response loss and reconstruction without recapturing", async () => { + const takeSnapshot = vi.fn(async () => { + throw new Error("response aborted after capture and retirement"); + }); + const recoverSnapshotReceipt = vi.fn().mockRejectedValueOnce(new Error("network offline")); + const stopSandbox = vi.fn(async () => ({ success: true as const })); + const f = fixture( + provider({ + name: "modal-vm", + capabilities: { ...provider().capabilities, snapshotStopsSandbox: true }, + takeSnapshot, + recoverSnapshotReceipt, + stopSandbox, + }) + ); + await readyFinite(f); + await f.shutdown.requestShutdown("checkpoint"); + f.shutdown.prepared(preparedEvent(f.store.value!)); + await f.shutdown.handleAlarm(); + expect(f.store.value).toMatchObject({ phase: "unknown", captureReceiptPending: true }); + expect(stopSandbox).not.toHaveBeenCalled(); + + recoverSnapshotReceipt.mockResolvedValue({ imageId: "newest-image" }); + f.setNow(f.store.value!.captureByMs! + 60_000); // Original request deadline is gone. + const restarted = new SandboxShutdownCoordinator(f.deps as never); + await restarted.handleAlarm(); + + expect(f.store.value).toMatchObject({ + phase: "saved", + sourceRetired: true, + captureReceiptPending: false, + receipt: { artifactId: "newest-image", provider: "modal-vm" }, + }); + expect(f.deps.sandbox.recordSandboxSnapshot).toHaveBeenCalledWith( + GENERATION.sandboxId, + "newest-image", + "runtime-1" + ); + expect(takeSnapshot).toHaveBeenCalledOnce(); + expect(stopSandbox).toHaveBeenCalledOnce(); + expect(f.deps.sandbox.recordSandboxSnapshot.mock.invocationCallOrder[0]).toBeLessThan( + stopSandbox.mock.invocationCallOrder[0] + ); + }); + + it("does not publish a recovered capture for a replaced generation", async () => { + const recoverSnapshotReceipt = vi.fn(async () => { + f.sandboxRow.created_at += 1; + return { imageId: "stale-image" }; + }); + const f = fixture(provider({ recoverSnapshotReceipt })); + await readyFinite(f); + await f.shutdown.requestShutdown("checkpoint"); + f.store.write({ ...f.store.value!, phase: "unknown", captureReceiptPending: true }); + await f.shutdown.handleAlarm(); + expect(f.deps.sandbox.recordSandboxSnapshot).not.toHaveBeenCalled(); + expect(f.store.value?.receipt).toBeUndefined(); + }); + it("distinguishes unmanaged and held shutdown requests", async () => { const f = fixture(); diff --git a/packages/control-plane/src/session/sandbox-shutdown.ts b/packages/control-plane/src/session/sandbox-shutdown.ts index 32d72dcf92..813aceb6fc 100644 --- a/packages/control-plane/src/session/sandbox-shutdown.ts +++ b/packages/control-plane/src/session/sandbox-shutdown.ts @@ -32,6 +32,7 @@ const STOP_MS = 60_000; const CAPTURE_MS = 300_000; const RETIRE_MS = 30_000; const MARGIN_MS = 30_000; +const CAPTURE_RECEIPT_RETRY_MS = 60_000; class ShutdownDeadlineError extends Error {} @@ -58,6 +59,7 @@ interface ShutdownDependencies { /** One durable owner of planned stopping. Provider side effects never imply a saved receipt. */ export class SandboxShutdownCoordinator { private activeOperation: string | null = null; + private recoveringReceipt = false; private checkpointOperationId: string | null = null; private checkpointGeneration: SandboxGeneration | null = null; private retiringOperation: string | null = null; @@ -714,6 +716,10 @@ export class SandboxShutdownCoordinator { if (state.phase === "saved") return this.continuationPaused(state) ? "hold_watchdogs" : "continue"; await this.advance(); + const current = this.deps.store.read(); + if (current?.phase === "unknown" && current.captureReceiptPending) { + await this.recoverCaptureReceipt(current); + } return "hold_watchdogs"; } @@ -789,7 +795,12 @@ export class SandboxShutdownCoordinator { return; } this.activeOperation = state.operationId!; - const capturing = { ...state, phase: "capturing" as const }; + const capturing = { + ...state, + phase: "capturing" as const, + captureReceiptPending: + !!provider.capabilities.snapshotStopsSandbox && !!provider.recoverSnapshotReceipt, + }; this.publish(capturing); await this.deps.alarm.schedule(state.captureByMs!); try { @@ -823,35 +834,11 @@ export class SandboxShutdownCoordinator { sourceStopped = result.sourceStopped; } if (!this.owns(capturing)) return; - const receipt = { - kind: retained ? ("retained" as const) : ("snapshot" as const), + const retiring = this.commitCaptureReceipt( + capturing, artifactId, - provider: provider.name, - savedAtMs: this.now(), - runtimeVersion: this.deps.sandbox.getSandbox()?.runtime_version ?? null, - }; - const retiring: ShutdownRecord = { - ...capturing, - phase: "retiring", - receipt, - savedAtMs: receipt.savedAtMs, - }; - // Receipt and legacy projection describe the same capture. Either both - // commit for this generation or neither may authorize source retirement. - this.deps.session.transaction(() => { - if (!this.owns(capturing)) throw new Error("Snapshot generation was superseded"); - if ( - !retained && - !this.deps.sandbox.recordSandboxSnapshot( - state.generation.sandboxId, - artifactId, - receipt.runtimeVersion - ) - ) - throw new Error("Snapshot generation was superseded"); - this.deps.store.write(retiring); - }); - this.announce(retiring); + retained ? "retained" : "snapshot" + ); if (sourceStopped) this.finish(retiring); else await this.retire(retiring); } catch (error) { @@ -868,6 +855,85 @@ export class SandboxShutdownCoordinator { } } + private commitCaptureReceipt( + state: ShutdownRecord, + artifactId: string, + kind: "retained" | "snapshot" + ): ShutdownRecord { + const receipt = { + kind, + artifactId, + provider: this.deps.provider.name, + savedAtMs: this.now(), + runtimeVersion: this.deps.sandbox.getSandbox()?.runtime_version ?? null, + }; + const retiring: ShutdownRecord = { + ...state, + phase: "retiring", + captureReceiptPending: false, + error: undefined, + receipt, + savedAtMs: receipt.savedAtMs, + }; + // Receipt and legacy projection describe the same capture. Either both + // commit for this generation or neither may authorize source retirement. + this.deps.session.transaction(() => { + if (!this.owns(state)) throw new Error("Snapshot generation was superseded"); + if ( + kind === "snapshot" && + !this.deps.sandbox.recordSandboxSnapshot( + state.generation.sandboxId, + artifactId, + receipt.runtimeVersion + ) + ) + throw new Error("Snapshot generation was superseded"); + this.deps.store.write(retiring); + }); + this.announce(retiring); + return retiring; + } + + private async recoverCaptureReceipt(state: ShutdownRecord): Promise { + const recover = this.deps.provider.recoverSnapshotReceipt; + if ( + this.recoveringReceipt || + !recover || + !state.providerObjectId || + state.restoreInvoked || + !this.current(state) || + !this.providerMatches(state) + ) + return; + this.recoveringReceipt = true; + try { + // A read-only lookup is safe after the capture deadline or eviction. Persist + // its next wakeup first; absence is not permission to repeat the capture. + await this.deps.alarm.schedule(this.now() + CAPTURE_RECEIPT_RETRY_MS); + const deadlineAtMs = this.now() + RETIRE_MS; + const session = this.deps.session.getSession()!; + const receipt = await this.bounded(deadlineAtMs, (signal) => + recover.call(this.deps.provider, { + providerObjectId: state.providerObjectId!, + sessionId: session.session_name || session.id, + deadlineAtMs, + signal, + }) + ); + if (!receipt || !this.owns(state)) return; + const retiring = this.commitCaptureReceipt( + { ...state, retireByMs: this.now() + RETIRE_MS }, + receipt.imageId, + "snapshot" + ); + await this.retire(retiring); + } catch (error) { + this.deps.log?.warn("Snapshot receipt recovery remains unconfirmed", { error }); + } finally { + this.recoveringReceipt = false; + } + } + private async retire(state: ShutdownRecord): Promise { if (this.retiringOperation === state.operationId) return; if (!state.receipt || !state.providerObjectId) return; diff --git a/packages/control-plane/src/session/schema.ts b/packages/control-plane/src/session/schema.ts index 9b05dcc9dc..dc5a4a45ac 100644 --- a/packages/control-plane/src/session/schema.ts +++ b/packages/control-plane/src/session/schema.ts @@ -202,6 +202,7 @@ CREATE TABLE IF NOT EXISTS sandbox ( boot_phase TEXT, -- JSON SandboxBootPhase the runtime last reported; NULL once ready boot_seq INTEGER, -- Sequence of that report, for de-duplicating resends fenced INTEGER NOT NULL DEFAULT 0, -- 1 once the generation's credentials were revoked for good (boot budget) + startup_rejected INTEGER NOT NULL DEFAULT 0, -- rejected allocation retains a cleanup obligation created_at INTEGER NOT NULL ); @@ -723,6 +724,11 @@ export const MIGRATIONS: readonly SchemaMigration[] = [ singleton INTEGER PRIMARY KEY CHECK (singleton = 1), state TEXT NOT NULL )`, }, + { + id: 55, + description: "Retain rejected sandbox startup cleanup intent", + run: "ALTER TABLE sandbox ADD COLUMN startup_rejected INTEGER NOT NULL DEFAULT 0", + }, ]; function removePersistedHookOutputTails(sql: SqlStorage): void { diff --git a/packages/control-plane/src/session/types.ts b/packages/control-plane/src/session/types.ts index d29064454c..bd64c47ebb 100644 --- a/packages/control-plane/src/session/types.ts +++ b/packages/control-plane/src/session/types.ts @@ -213,6 +213,7 @@ export const sandboxRowSchema = z.object({ * cannot self-heal the way a watchdog-failed one may. */ fenced: z.number(), + startup_rejected: z.number().default(0), created_at: z.number(), }); diff --git a/packages/control-plane/src/session/websocket-manager.test.ts b/packages/control-plane/src/session/websocket-manager.test.ts index fcfcadccdc..c340989991 100644 --- a/packages/control-plane/src/session/websocket-manager.test.ts +++ b/packages/control-plane/src/session/websocket-manager.test.ts @@ -208,6 +208,7 @@ function createSandboxRow(modalSandboxId: string): SandboxRow { boot_phase: null, boot_seq: null, fenced: 0, + startup_rejected: 0, created_at: Date.now(), }; } diff --git a/packages/control-plane/test/integration/sandbox-shutdown.test.ts b/packages/control-plane/test/integration/sandbox-shutdown.test.ts index 154e9b869f..93c1199311 100644 --- a/packages/control-plane/test/integration/sandbox-shutdown.test.ts +++ b/packages/control-plane/test/integration/sandbox-shutdown.test.ts @@ -77,6 +77,70 @@ async function readShutdown(stub: DurableObjectStub): Promise { + it("recovers an interrupted VM capture through the lifecycle boundary and commits before retirement", async () => { + const { stub } = await initNamedSession(`vm-capture-receipt-${Date.now()}`); + await seedSandboxAuth(stub, { authToken: AUTH_TOKEN, sandboxId: SANDBOX_ID, status: "ready" }); + await runInSessionDO(stub, (_instance, state) => { + state.storage.sql.exec("UPDATE sandbox SET modal_object_id = 'sb-captured'"); + }); + const now = Date.now(); + await seedShutdown(stub, { + phase: "capturing", + provider: "modal-vm", + providerObjectId: "sb-captured", + operationId: "lost-terminal-capture", + captureReceiptPending: true, + stopByMs: now - 120_000, + captureByMs: now - 60_000, + retireByMs: now - 30_000, + generationReady: true, + lifecyclePolicy: "confirmed", + protocolVersion: 1, + }); + const evidence = await runInSessionDO(stub, async (instance, durableState) => { + let lookupCount = 0; + let stopCount = 0; + const provider: SandboxProvider = { + name: "modal-vm", + capabilities: { + supportsSandboxTimeout: true, + supportsSnapshots: true, + snapshotStopsSandbox: true, + supportsRestore: true, + supportsExplicitStop: true, + }, + createSandbox: async () => { + throw new Error("must not create"); + }, + takeSnapshot: async () => { + throw new Error("must not recapture"); + }, + recoverSnapshotReceipt: async ({ providerObjectId }) => { + expect(providerObjectId).toBe("sb-captured"); + lookupCount++; + return { imageId: "im-newest-recovered" }; + }, + stopSandbox: async () => { + const row = durableState.storage.sql.exec("SELECT snapshot_image_id FROM sandbox").one(); + expect(row.snapshot_image_id).toBe("im-newest-recovered"); + stopCount++; + return { success: true }; + }, + }; + const restarted = realLifecycleHarness(instance, durableState, provider); + await restarted.manager.handleShutdownAlarm(); + return { lookupCount, stopCount, snapshot: restarted.manager.shutdownSnapshot() }; + }); + expect(evidence).toMatchObject({ + lookupCount: 1, + stopCount: 1, + snapshot: { phase: "saved", hasRecoveryPoint: true }, + }); + expect(await queryDO(stub, "SELECT snapshot_image_id, status FROM sandbox")).toEqual([ + { snapshot_image_id: "im-newest-recovered", status: "stopped" }, + ]); + }); + it("preserves a completed session status when shutdown begins between prompts", async () => { const name = `shutdown-completed-status-${Date.now()}`; const { stub } = await initNamedSession(name); diff --git a/packages/modal-infra/deploy.py b/packages/modal-infra/deploy.py index fd21c60452..49ec2405ab 100644 --- a/packages/modal-infra/deploy.py +++ b/packages/modal-infra/deploy.py @@ -65,6 +65,18 @@ def _publish_image_record(record: dict[str, str]) -> None: path.write_text(json.dumps(record) + "\n") +def _deployed_vm_image() -> str | None: + """Retain the existing VM capability until the consumer selector has switched.""" + try: + image_id = modal.Function.from_name(APP_NAME, "deployment_vm_image").remote() + except modal.exception.NotFoundError: + # First deployment, or a pre-VM app. Other failures must block deployment. + return None + if image_id is not None and (not isinstance(image_id, str) or not image_id.strip()): + raise RuntimeError("Deployed VM image reference is invalid") + return image_id + + def build_sandbox_image(*, with_docker: bool = False) -> None: """Build the image used by dynamic sandboxes before requests can create them. @@ -97,6 +109,10 @@ def build_sandbox_image(*, with_docker: bool = False) -> None: "imageId": base_image.object_id, "buildHash": base_image_plan["buildHash"], } + if not with_docker: + retained_vm_image = _deployed_vm_image() + if retained_vm_image: + record["dockerImageId"] = retained_vm_image _publish_image_record(record) if not with_docker: return diff --git a/packages/modal-infra/src/sandbox/launch_policy.py b/packages/modal-infra/src/sandbox/launch_policy.py index 05c753fd14..23546473d2 100644 --- a/packages/modal-infra/src/sandbox/launch_policy.py +++ b/packages/modal-infra/src/sandbox/launch_policy.py @@ -99,14 +99,14 @@ def _identity_digest(*parts: str) -> str: return hashlib.sha256("\n".join(parts).encode()).hexdigest() -def docker_allocation_name(session_id: str, sandbox_id: str) -> str: - """Deterministic, Modal-safe sandbox name for one control-plane generation. +def docker_allocation_name(session_id: str) -> str: + """One provider-enforced running allocation slot per session. - `sandbox_id` already embeds the generation timestamp, so the pair names - exactly one launch attempt. Modal names are limited to 64 chars of - `[A-Za-z0-9._-]`, which control-plane identifiers do not satisfy directly. + Generations retain distinct ownership tags, not distinct names. A missing + lookup cannot authorize overlapping creates: Modal rejects a conflicting + name until the previous sandbox has completely stopped. """ - return ALLOCATION_NAME_PREFIX + _identity_digest("modal-vm", session_id, sandbox_id)[:40] + return ALLOCATION_NAME_PREFIX + _identity_digest("modal-vm", session_id)[:40] def docker_allocation_tags(session_id: str, sandbox_id: str) -> dict[str, str]: diff --git a/packages/modal-infra/src/sandbox/manager.py b/packages/modal-infra/src/sandbox/manager.py index 155fce3c30..887592f61c 100644 --- a/packages/modal-infra/src/sandbox/manager.py +++ b/packages/modal-infra/src/sandbox/manager.py @@ -34,6 +34,7 @@ VNC_PASSWORD_MAX_BYTES, VNC_PORT, ) +from sandbox_runtime.docker_control import CONTROL_TIMEOUT_SECONDS from sandbox_runtime.log_config import get_logger from sandbox_runtime.types import SandboxStatus, SessionConfig @@ -555,14 +556,13 @@ async def _launch_docker_sandbox( ) -> tuple[modal.Sandbox, bool]: """Create a Docker VM under a deterministic name, adopting an existing one. - VM creation can outlive the control plane's HTTP request. Naming each - generation's allocation lets a retried create adopt the sandbox Modal - already made instead of starting a second VM, and lets the next - generation retire a predecessor whose object id was never learned. + VM creation can outlive the control plane's HTTP request. One name per + session serializes generations at Modal even when a predecessor lookup + misses an in-flight create. Only matching generation tags permit adoption. """ if retire_sandbox_id: await self._retire_docker_allocation(session_id, retire_sandbox_id) - name = docker_allocation_name(session_id, sandbox_id) + name = docker_allocation_name(session_id) tags = docker_allocation_tags(session_id, sandbox_id) existing = await self._find_owned_docker_allocation(name, tags) if existing is None: @@ -630,7 +630,7 @@ async def _find_owned_docker_allocation( async def _retire_docker_allocation(self, session_id: str, sandbox_id: str) -> None: """Terminate a prior generation's named VM, only when its ownership tags match.""" - name = docker_allocation_name(session_id, sandbox_id) + name = docker_allocation_name(session_id) try: sandbox = await modal.Sandbox.from_name.aio(APP_NAME, name) except modal.exception.NotFoundError: @@ -727,7 +727,7 @@ async def take_snapshot( "-m", "sandbox_runtime.docker_control", "prepare", - timeout=min(snapshot_timeout_seconds, 45), + timeout=min(snapshot_timeout_seconds, CONTROL_TIMEOUT_SECONDS), ) if await probe.wait.aio() != 0: raise RuntimeError("Modal VM Docker shutdown preparation was not confirmed") @@ -755,7 +755,16 @@ async def take_snapshot( return image_id async def stop_sandbox(self, sandbox_id: str) -> None: - """Terminate a provider sandbox by its immutable Modal object id.""" + """Resolve a recovery reference if needed, then confirm immutable-ID retirement.""" + if sandbox_id.startswith("modal-vm-session:"): + from .terminal_snapshot import recorded_vm_source + + source_id = await recorded_vm_source(sandbox_id) + if source_id is None: + handle = await self.get_sandbox_by_id(sandbox_id) + assert handle is not None and handle.modal_object_id is not None + source_id = handle.modal_object_id + sandbox_id = source_id try: sandbox = await modal.Sandbox.from_id.aio(sandbox_id) await sandbox.terminate.aio(wait=True) @@ -765,32 +774,49 @@ async def stop_sandbox(self, sandbox_id: str) -> None: async def get_sandbox_by_id(self, sandbox_id: str) -> SandboxHandle | None: """ - Get a sandbox handle by its ID. - - Uses Modal's Sandbox.from_id() to retrieve an existing sandbox. + Get a sandbox by immutable ID or a generation-checked pending reference. Args: - sandbox_id: The Modal sandbox ID + sandbox_id: The Modal sandbox ID or opaque VM session reference Returns: - SandboxHandle if found, None otherwise + SandboxHandle if found, None for a confirmed missing immutable ID. + Missing pending references remain ambiguous and raise an error. """ - try: - modal_sandbox = await modal.Sandbox.from_id.aio(sandbox_id) - tags = await modal_sandbox.get_tags.aio() - backend = tags.get("openinspect_backend", "modal") - if backend not in ("modal", "modal-vm"): - raise ValueError("Unknown sandbox backend tag") - return SandboxHandle( - sandbox_backend="modal-vm" if backend == "modal-vm" else "modal", - sandbox_id=sandbox_id, - modal_sandbox=modal_sandbox, - status=SandboxStatus.READY, # Assume ready if we can retrieve it - created_at=time.time(), + if sandbox_id.startswith("modal-vm-session:"): + identity = json.loads(sandbox_id.removeprefix("modal-vm-session:")) + if ( + not isinstance(identity, list) + or len(identity) != 2 + or not all(isinstance(part, str) and part for part in identity) + ): + raise ValueError("Invalid pending VM reference") + session_id, generation_id = identity + modal_sandbox = await self._find_owned_docker_allocation( + docker_allocation_name(session_id), + docker_allocation_tags(session_id, generation_id), ) - except Exception as e: - log.warn("sandbox.lookup_error", sandbox_id=sandbox_id, exc=e) - return None + if modal_sandbox is None: + # An in-flight create can still materialize. Never report confirmed + # absence/retirement for an unresolved launch intent. + raise RuntimeError("VM launch identity is not yet visible") + else: + try: + modal_sandbox = await modal.Sandbox.from_id.aio(sandbox_id) + except modal.exception.NotFoundError: + return None + tags = await modal_sandbox.get_tags.aio() + backend = tags.get("openinspect_backend", "modal") + if backend not in ("modal", "modal-vm"): + raise ValueError("Unknown sandbox backend tag") + return SandboxHandle( + sandbox_backend="modal-vm" if backend == "modal-vm" else "modal", + sandbox_id=sandbox_id, + modal_object_id=modal_sandbox.object_id, + modal_sandbox=modal_sandbox, + status=SandboxStatus.READY, # Assume ready if we can retrieve it + created_at=time.time(), + ) async def restore_from_snapshot( self, diff --git a/packages/modal-infra/src/sandbox/terminal_snapshot.py b/packages/modal-infra/src/sandbox/terminal_snapshot.py new file mode 100644 index 0000000000..7583c0d7e7 --- /dev/null +++ b/packages/modal-infra/src/sandbox/terminal_snapshot.py @@ -0,0 +1,47 @@ +"""Retryable terminal VM capture, keyed by the generation's stable source reference. + +Modal Dict receipts survive function restarts and are retained for seven days +without access. A captured image is recorded before source retirement. An +unfinished capture intent is deliberately not retried: its outcome is unknown. +""" + +import modal + +from ..app_config import APP_NAME +from .manager import SandboxManager + +_receipts = modal.Dict.from_name(f"{APP_NAME}-vm-terminal-snapshots", create_if_missing=True) + + +async def recover_vm_snapshot(reference: str) -> str | None: + """Read an existing capture receipt; never capture or retire any source.""" + receipt = await _receipts.get.aio(reference) + return receipt.get("image_id") if receipt else None + + +async def recorded_vm_source(reference: str) -> str | None: + """Resolve a launch reference after capture, even when its named source is gone.""" + receipt = await _receipts.get.aio(reference) + return receipt["source_id"] if receipt else None + + +async def snapshot_vm(manager: SandboxManager, reference: str, timeout_seconds: float) -> str: + """Recover a prior receipt or capture once, then confirm source retirement.""" + receipt = await _receipts.get.aio(reference) + if receipt is None: + handle = await manager.get_sandbox_by_id(reference) + if handle is None or handle.sandbox_backend != "modal-vm": + raise RuntimeError("Terminal capture requires a confirmed VM source") + intent = {"source_id": handle.modal_object_id, "image_id": None} + if await _receipts.put.aio(reference, intent, skip_if_exists=True): + image_id = await manager.take_snapshot(handle, timeout_seconds=timeout_seconds) + receipt = {**intent, "image_id": image_id} + # Never retire if this write fails or its outcome is unknown. + await _receipts.put.aio(reference, receipt) + else: + receipt = await _receipts.get.aio(reference) + recorded_image_id = receipt.get("image_id") if receipt else None + if not isinstance(recorded_image_id, str) or not recorded_image_id: + raise RuntimeError("Terminal capture is in progress or its result is unconfirmed") + await manager.stop_sandbox(receipt["source_id"]) + return recorded_image_id diff --git a/packages/modal-infra/src/web_api.py b/packages/modal-infra/src/web_api.py index 6ec4dc9b4a..bd576f48ae 100644 --- a/packages/modal-infra/src/web_api.py +++ b/packages/modal-infra/src/web_api.py @@ -488,6 +488,16 @@ async def api_create_sandbox( } +@app.function(image=function_image) +def deployment_vm_image() -> str | None: + """Private SDK-only deployment handshake; preserve capability across selector cutover.""" + import os + + from .images.base import DOCKER_IMAGE_ID_ENV + + return os.environ.get(DOCKER_IMAGE_ID_ENV) + + @app.function(image=function_image) @fastapi_endpoint(method="GET") def api_health() -> dict: @@ -545,10 +555,6 @@ async def api_snapshot_sandbox( manager = SandboxManager() - handle = await manager.get_sandbox_by_id(sandbox_id) - if not handle: - raise HTTPException(status_code=404, detail=f"Sandbox not found: {sandbox_id}") - deadline_at_ms = request.get("deadline_at_ms") timeout_seconds = SNAPSHOT_FILESYSTEM_TIMEOUT_SECONDS if deadline_at_ms is not None: @@ -561,17 +567,29 @@ async def api_snapshot_sandbox( timeout_seconds = (deadline_at_ms / 1000) - time.time() if timeout_seconds <= 0: raise HTTPException(status_code=408, detail="snapshot deadline expired") - source_stopped = handle.sandbox_backend == "modal-vm" + source_stopped = request.get("sandbox_backend") == "modal-vm" try: - # Include Docker preparation, capture and confirmed retirement in one budget. - # Timeout is an unknown outcome, never a successful source-stopped receipt. async with asyncio.timeout(timeout_seconds): - if deadline_at_ms is None: - image_id = await manager.take_snapshot(handle) - else: - image_id = await manager.take_snapshot(handle, timeout_seconds=timeout_seconds) if source_stopped: - await handle.modal_sandbox.terminate.aio(wait=True) + from .sandbox.terminal_snapshot import snapshot_vm + + image_id = await snapshot_vm(manager, sandbox_id, timeout_seconds) + else: + handle = await manager.get_sandbox_by_id(sandbox_id) + if not handle: + raise HTTPException( + status_code=404, detail=f"Sandbox not found: {sandbox_id}" + ) + if handle.sandbox_backend == "modal-vm": + raise HTTPException( + status_code=400, detail="VM capture requires sandbox_backend" + ) + if deadline_at_ms is None: + image_id = await manager.take_snapshot(handle) + else: + image_id = await manager.take_snapshot( + handle, timeout_seconds=timeout_seconds + ) except (TimeoutError, ModalTimeoutError) as exc: raise HTTPException(status_code=408, detail="snapshot deadline expired") from exc return { @@ -584,6 +602,27 @@ async def api_snapshot_sandbox( } +@app.function(image=function_image, secrets=[internal_api_secret]) +@fastapi_endpoint(method="POST") +async def api_recover_sandbox_snapshot( + request: dict[str, Any], + authorization: str | None = Header(None), +) -> dict[str, Any]: + """Read a terminal VM capture receipt without repeating any provider side effect.""" + async with _execute_endpoint( + endpoint_name="api_recover_sandbox_snapshot", + authorization=authorization, + trace_id=None, + request_id=None, + ): + sandbox_id = request.get("sandbox_id") + if not isinstance(sandbox_id, str) or not sandbox_id: + raise HTTPException(status_code=400, detail="sandbox_id is required") + from .sandbox.terminal_snapshot import recover_vm_snapshot + + return {"success": True, "data": {"image_id": await recover_vm_snapshot(sandbox_id)}} + + @app.function(image=function_image, secrets=[internal_api_secret]) @fastapi_endpoint(method="POST") async def api_stop_sandbox( diff --git a/packages/modal-infra/tests/test_deploy.py b/packages/modal-infra/tests/test_deploy.py index e7ac2a5470..65fb4e7c12 100644 --- a/packages/modal-infra/tests/test_deploy.py +++ b/packages/modal-infra/tests/test_deploy.py @@ -8,9 +8,18 @@ from unittest.mock import Mock import deploy +import modal import pytest +@pytest.fixture(autouse=True) +def deployed_vm_reference(monkeypatch): + function = Mock() + function.remote.return_value = None + monkeypatch.setattr(deploy.modal.Function, "from_name", Mock(return_value=function)) + return function + + def test_deployment_rejects_missing_image_without_opt_in(monkeypatch, tmp_path) -> None: from src.images import base @@ -108,7 +117,11 @@ def test_eager_build_does_not_register_functions_before_image_exists() -> None: assert result.returncode == 0, result.stderr -def test_build_sandbox_image_eagerly_builds_against_deployed_app(monkeypatch, tmp_path) -> None: +@pytest.mark.parametrize("prior_vm_image", [None, "im-existing-vm"]) +def test_build_sandbox_image_eagerly_builds_against_deployed_app( + monkeypatch, tmp_path, deployed_vm_reference, prior_vm_image +) -> None: + deployed_vm_reference.remote.return_value = prior_vm_image deployed_app = object() lookup = Mock(return_value=deployed_app) build = Mock() @@ -135,10 +148,25 @@ def test_build_sandbox_image_eagerly_builds_against_deployed_app(monkeypatch, tm build.assert_called_once_with(deployed_app) assert create.call_args.kwargs["env"] is plan["runtimeEnv"] sandbox.terminate.assert_called_once() - assert json.loads((tmp_path / "selected.json").read_text()) == { + expected = { "imageId": "im-verified", "buildHash": "packed-recipe", } + if prior_vm_image: + expected["dockerImageId"] = prior_vm_image + assert json.loads((tmp_path / "selected.json").read_text()) == expected + assert create.call_count == 1 # Reverse cutover retains capability without another VM build. + + +def test_reverse_cutover_fails_closed_if_existing_capability_cannot_be_read(deployed_vm_reference): + deployed_vm_reference.remote.side_effect = RuntimeError("lookup unavailable") + with pytest.raises(RuntimeError, match="lookup unavailable"): + deploy._deployed_vm_image() + + +def test_first_deployment_has_no_vm_capability(deployed_vm_reference): + deployed_vm_reference.remote.side_effect = modal.exception.NotFoundError("not deployed") + assert deploy._deployed_vm_image() is None def test_local_base_image_retains_its_packed_plan(monkeypatch, tmp_path) -> None: diff --git a/packages/modal-infra/tests/test_docker_launch.py b/packages/modal-infra/tests/test_docker_launch.py index 44b371f2f2..73b743cfd5 100644 --- a/packages/modal-infra/tests/test_docker_launch.py +++ b/packages/modal-infra/tests/test_docker_launch.py @@ -70,16 +70,10 @@ def test_docker_base_image_requires_provisioning(monkeypatch): assert docker_base_image() is sentinel -def test_allocation_name_is_deterministic_per_generation_and_modal_safe(): - name = docker_allocation_name("session/with:odd chars", "sandbox-acme-repo-1700000000000") - - assert name == docker_allocation_name( - "session/with:odd chars", "sandbox-acme-repo-1700000000000" - ) - assert name != docker_allocation_name( - "session/with:odd chars", "sandbox-acme-repo-1700000000001" - ) - assert name != docker_allocation_name("other-session", "sandbox-acme-repo-1700000000000") +def test_allocation_name_is_stable_per_session_and_modal_safe(): + name = docker_allocation_name("session/with:odd chars") + assert name == docker_allocation_name("session/with:odd chars") + assert name != docker_allocation_name("other-session") assert len(name) <= 64 assert re.fullmatch(r"[a-zA-Z0-9-_.]+", name) assert not re.fullmatch(r"ap-[a-zA-Z0-9]{22}", name) diff --git a/packages/modal-infra/tests/test_sandbox_launch.py b/packages/modal-infra/tests/test_sandbox_launch.py index 02e1cd442d..9ec41573d7 100644 --- a/packages/modal-infra/tests/test_sandbox_launch.py +++ b/packages/modal-infra/tests/test_sandbox_launch.py @@ -305,7 +305,7 @@ async def test_docker_launch_selects_vm_runtime_and_named_allocation(monkeypatch assert kwargs["experimental_options"] == {"vm_runtime": True} assert kwargs["cpu"] == 2.0 assert kwargs["memory"] == 4096 - assert kwargs["name"] == docker_allocation_name("session-1", "sandbox-acme-repo-1700000000000") + assert kwargs["name"] == docker_allocation_name("session-1") assert kwargs["tags"] == docker_allocation_tags("session-1", "sandbox-acme-repo-1700000000000") # The trusted signal wins over any user-supplied value. assert kwargs["env"][DOCKER_ENABLED_ENV_VAR] == "true" @@ -348,9 +348,7 @@ async def test_docker_launch_adopts_an_existing_owned_allocation(monkeypatch): assert "kwargs" not in captured assert handle.modal_object_id == "modal-existing" - from_name.assert_awaited_once_with( - "open-inspect", docker_allocation_name("session-1", "sandbox-acme-repo-1700000000000") - ) + from_name.assert_awaited_once_with("open-inspect", docker_allocation_name("session-1")) @pytest.mark.asyncio @@ -478,10 +476,10 @@ async def test_docker_launch_retires_the_prior_generation_only_when_owned(monkey ) prior.get_tags.aio = prior.get_tags prior.terminate.aio = prior.terminate - prior_name = docker_allocation_name("session-1", "sandbox-acme-repo-1699999999999") + prior_name = docker_allocation_name("session-1") async def from_name(_app, name): - if name == prior_name: + if name == prior_name and not prior.terminate.await_count: return prior _not_found() @@ -494,20 +492,53 @@ async def from_name(_app, name): ) prior.terminate.assert_awaited_once_with(wait=True) - assert captured["kwargs"]["name"] == docker_allocation_name( - "session-1", "sandbox-acme-repo-1700000000000" - ) + assert captured["kwargs"]["name"] == docker_allocation_name("session-1") # A prior allocation with foreign tags is left alone. prior.terminate.reset_mock() prior.get_tags = AsyncMock(return_value={"openinspect_kind": "other"}) prior.get_tags.aio = prior.get_tags - await manager.create_sandbox( - _docker_config(retire_sandbox_id="sandbox-acme-repo-1699999999999") - ) + with pytest.raises(RuntimeError, match="ownership mismatch"): + await manager.create_sandbox( + _docker_config(retire_sandbox_id="sandbox-acme-repo-1699999999999") + ) prior.terminate.assert_not_awaited() +@pytest.mark.asyncio +async def test_late_predecessor_cannot_materialize_beside_successor(monkeypatch): + from modal.exception import AlreadyExistsError, NotFoundError + + manager, _, _ = _docker_manager(monkeypatch) + predecessor_name = docker_allocation_name("session-1") + predecessor = SimpleNamespace( + object_id="late-predecessor", + get_tags=SimpleNamespace( + aio=AsyncMock(return_value=docker_allocation_tags("session-1", "prior")) + ), + ) + lookup = AsyncMock( + side_effect=[NotFoundError("still creating"), NotFoundError("still creating"), predecessor] + ) + monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.from_name", SimpleNamespace(aio=lookup)) + + async def create(kwargs, *, repository_image): + # Provider-side naming wins the race after both client lookups missed it. + if kwargs["name"] == predecessor_name: + raise AlreadyExistsError("predecessor won the name") + return SimpleNamespace(object_id="duplicate-successor") + + monkeypatch.setattr("src.sandbox.manager._create_sandbox", create) + with pytest.raises(RuntimeError, match="ownership mismatch"): + await manager._launch_docker_sandbox( + session_id="session-1", + sandbox_id="successor", + retire_sandbox_id="prior", + create_kwargs={}, + repository_image=False, + ) + + @pytest.mark.asyncio @pytest.mark.parametrize("termination_fails", [False, True]) async def test_docker_successor_waits_for_confirmed_predecessor_retirement( diff --git a/packages/modal-infra/tests/test_snapshot_timeout.py b/packages/modal-infra/tests/test_snapshot_timeout.py index cdd2a41f7e..49779f51fb 100644 --- a/packages/modal-infra/tests/test_snapshot_timeout.py +++ b/packages/modal-infra/tests/test_snapshot_timeout.py @@ -7,7 +7,9 @@ import pytest from modal.exception import NotFoundError as ModalNotFoundError +from sandbox_runtime.docker_control import CONTROL_TIMEOUT_SECONDS from sandbox_runtime.types import SandboxStatus +from src.sandbox.launch_policy import docker_allocation_tags from src.sandbox.manager import ( SNAPSHOT_FILESYSTEM_TIMEOUT_SECONDS, SandboxHandle, @@ -15,6 +17,47 @@ ) +@pytest.mark.asyncio +async def test_pending_vm_reference_recovers_owned_allocation(monkeypatch): + sandbox = SimpleNamespace( + object_id="sb-owned", + get_tags=_async_method(docker_allocation_tags("session", "generation")), + ) + lookup = _async_method(sandbox) + monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.from_name", lookup) + handle = await SandboxManager().get_sandbox_by_id('modal-vm-session:["session","generation"]') + assert handle is not None and handle.modal_sandbox is sandbox + assert handle.sandbox_backend == "modal-vm" + + +@pytest.mark.asyncio +async def test_pending_vm_reference_never_confirms_absence(monkeypatch): + monkeypatch.setattr( + "src.sandbox.terminal_snapshot.recorded_vm_source", AsyncMock(return_value=None) + ) + lookup = _async_method() + lookup.aio.side_effect = ModalNotFoundError("not visible yet") + monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.from_name", lookup) + with pytest.raises(RuntimeError, match="not yet visible"): + await SandboxManager().stop_sandbox('modal-vm-session:["session","generation"]') + + +@pytest.mark.asyncio +async def test_pending_vm_reference_cannot_stop_another_generation(monkeypatch): + monkeypatch.setattr( + "src.sandbox.terminal_snapshot.recorded_vm_source", AsyncMock(return_value=None) + ) + sandbox = SimpleNamespace( + object_id="sb-owned", + get_tags=_async_method(docker_allocation_tags("session", "other")), + terminate=_async_method(), + ) + monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.from_name", _async_method(sandbox)) + with pytest.raises(RuntimeError, match="ownership mismatch"): + await SandboxManager().stop_sandbox('modal-vm-session:["session","generation"]') + sandbox.terminate.aio.assert_not_awaited() + + def _async_method(return_value=None): method = MagicMock() method.aio = AsyncMock(return_value=return_value) @@ -42,7 +85,7 @@ async def test_take_snapshot_passes_explicit_timeout(): @pytest.mark.asyncio async def test_get_sandbox_by_id_awaits_async_lookup(monkeypatch): - modal_sandbox = SimpleNamespace(get_tags=_async_method({})) + modal_sandbox = SimpleNamespace(object_id="sb-owned", get_tags=_async_method({})) from_id = _async_method(modal_sandbox) monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.from_id", from_id) @@ -50,10 +93,37 @@ async def test_get_sandbox_by_id_awaits_async_lookup(monkeypatch): assert handle is not None assert handle.modal_sandbox is modal_sandbox + assert handle.modal_object_id == "sb-owned" from_id.assert_not_called() from_id.aio.assert_awaited_once_with("sandbox-1") +@pytest.mark.asyncio +async def test_tag_lookup_failure_is_not_sandbox_absence(monkeypatch): + tags = _async_method() + tags.aio.side_effect = RuntimeError("tag service unavailable") + monkeypatch.setattr( + "src.sandbox.manager.modal.Sandbox.from_id", + _async_method(SimpleNamespace(object_id="sb-owned", get_tags=tags)), + ) + with pytest.raises(RuntimeError, match="tag service unavailable"): + await SandboxManager().get_sandbox_by_id("sandbox-1") + + +@pytest.mark.asyncio +async def test_unknown_backend_tag_is_explicitly_rejected(monkeypatch): + monkeypatch.setattr( + "src.sandbox.manager.modal.Sandbox.from_id", + _async_method( + SimpleNamespace( + object_id="sb-owned", get_tags=_async_method({"openinspect_backend": "unknown"}) + ) + ), + ) + with pytest.raises(ValueError, match="Unknown sandbox backend tag"): + await SandboxManager().get_sandbox_by_id("sandbox-1") + + @pytest.mark.asyncio @pytest.mark.parametrize("budget, expected", [(179.8, 179), (500, 300)]) async def test_take_snapshot_bounds_whole_second_timeout(budget, expected): @@ -170,6 +240,7 @@ async def test_vm_capture_requires_docker_preparation(exit_code): "sandbox_runtime.docker_control", "prepare", ) + assert execute.aio.call_args.kwargs["timeout"] == CONTROL_TIMEOUT_SECONDS @pytest.mark.asyncio diff --git a/packages/modal-infra/tests/test_web_api_build_sandbox.py b/packages/modal-infra/tests/test_web_api_build_sandbox.py index 728c67e07c..4e11bd7a82 100644 --- a/packages/modal-infra/tests/test_web_api_build_sandbox.py +++ b/packages/modal-infra/tests/test_web_api_build_sandbox.py @@ -1,7 +1,7 @@ """Tests for the Modal provider-session image-build APIs.""" from types import SimpleNamespace -from unittest.mock import ANY, AsyncMock, MagicMock +from unittest.mock import ANY, AsyncMock, MagicMock, Mock import pytest from modal.exception import NotFoundError as ModalNotFoundError @@ -23,6 +23,156 @@ } +@pytest.fixture(autouse=True) +def terminal_snapshot_receipts(monkeypatch): + from src.sandbox import terminal_snapshot + + records = {} + + async def put(key, value, *, skip_if_exists=False): + if skip_if_exists and key in records: + return False + records[key] = value + return True + + store = SimpleNamespace( + get=SimpleNamespace(aio=AsyncMock(side_effect=records.get)), + put=SimpleNamespace(aio=AsyncMock(side_effect=put)), + ) + monkeypatch.setattr(terminal_snapshot, "_receipts", store) + return records + + +@pytest.mark.asyncio +@pytest.mark.parametrize("image_id", [None, "im-recovered"]) +async def test_snapshot_receipt_recovery_is_read_only( + monkeypatch, terminal_snapshot_receipts, image_id +): + monkeypatch.setattr(web_api, "require_auth", lambda _authorization: None) + terminal_snapshot_receipts["sb-vm"] = {"source_id": "sb-vm", "image_id": image_id} + manager = Mock() + monkeypatch.setattr("src.sandbox.manager.SandboxManager", manager) + result = await web_api.api_recover_sandbox_snapshot.get_raw_f()( + {"sandbox_id": "sb-vm"}, authorization="Bearer token" + ) + assert result == {"success": True, "data": {"image_id": image_id}} + manager.assert_not_called() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("reference", ["sb-vm", 'modal-vm-session:["session","generation"]']) +async def test_terminal_capture_uses_real_lookup_identity_and_replays_after_retirement( + monkeypatch, terminal_snapshot_receipts, reference +): + from src.sandbox.launch_policy import docker_allocation_tags + + monkeypatch.setattr(web_api, "require_auth", lambda _authorization: None) + retired = False + + async def terminate(*, wait): + nonlocal retired + assert wait is True + assert terminal_snapshot_receipts[reference]["image_id"] == "im-recovery" + retired = True + + sandbox = SimpleNamespace( + object_id="sb-vm", + get_tags=SimpleNamespace( + aio=AsyncMock(return_value=docker_allocation_tags("session", "generation")) + ), + terminate=SimpleNamespace(aio=AsyncMock(side_effect=terminate)), + ) + + async def from_id(object_id): + assert object_id == "sb-vm" + if retired: + raise ModalNotFoundError("retired") + return sandbox + + monkeypatch.setattr( + "src.sandbox.manager.modal.Sandbox.from_id", + SimpleNamespace(aio=AsyncMock(side_effect=from_id)), + ) + monkeypatch.setattr( + "src.sandbox.manager.modal.Sandbox.from_name", + SimpleNamespace(aio=AsyncMock(return_value=sandbox)), + ) + capture = AsyncMock(return_value="im-recovery") + monkeypatch.setattr(SandboxManager, "take_snapshot", capture) + request = {"sandbox_id": reference, "sandbox_backend": "modal-vm"} + first = await _call_generic_snapshot(request) + assert retired + assert terminal_snapshot_receipts[reference]["source_id"] == "sb-vm" + # The CP may know only a launch reference when BOTH launch and capture responses + # were lost. After receipt-only recovery, explicit retirement must use the + # recorded immutable source, never the now-reusable session name. + from_name = SimpleNamespace(aio=AsyncMock(side_effect=ModalNotFoundError("source gone"))) + monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.from_name", from_name) + recovered = await web_api.api_recover_sandbox_snapshot.get_raw_f()( + {"sandbox_id": reference}, authorization="Bearer token" + ) + assert recovered["data"]["image_id"] == "im-recovery" + await SandboxManager().stop_sandbox(reference) + from_name.aio.assert_not_awaited() + assert await _call_generic_snapshot(request) == first + capture.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_terminal_snapshot_retry_recovers_receipt_after_source_is_gone( + monkeypatch, terminal_snapshot_receipts +): + monkeypatch.setattr(web_api, "require_auth", lambda _authorization: None) + handle = SimpleNamespace(sandbox_backend="modal-vm", modal_object_id="sb-vm") + manager = SimpleNamespace( + get_sandbox_by_id=AsyncMock(return_value=handle), + take_snapshot=AsyncMock(return_value="im-recovery"), + stop_sandbox=AsyncMock(), + ) + monkeypatch.setattr("src.sandbox.manager.SandboxManager", lambda: manager) + request = {"sandbox_id": "sb-vm", "sandbox_backend": "modal-vm"} + first = await _call_generic_snapshot(request) # Simulate dropping this response. + manager.get_sandbox_by_id.return_value = None + recovered = await _call_generic_snapshot(request) + assert recovered == first + assert recovered["data"]["image_id"] == "im-recovery" + manager.take_snapshot.assert_awaited_once() + manager.get_sandbox_by_id.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_terminal_snapshot_does_not_retire_without_durable_receipt(monkeypatch): + from src.sandbox import terminal_snapshot + + monkeypatch.setattr(web_api, "require_auth", lambda _authorization: None) + terminal_snapshot._receipts.put.aio.side_effect = [True, RuntimeError("receipt unavailable")] + manager = SimpleNamespace( + get_sandbox_by_id=AsyncMock( + return_value=SimpleNamespace(sandbox_backend="modal-vm", modal_object_id="sb-vm") + ), + take_snapshot=AsyncMock(return_value="im-recovery"), + stop_sandbox=AsyncMock(), + ) + monkeypatch.setattr("src.sandbox.manager.SandboxManager", lambda: manager) + with pytest.raises(web_api.HTTPException): + await _call_generic_snapshot({"sandbox_id": "sb-vm", "sandbox_backend": "modal-vm"}) + manager.stop_sandbox.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_terminal_snapshot_does_not_repeat_unknown_capture( + monkeypatch, terminal_snapshot_receipts +): + monkeypatch.setattr(web_api, "require_auth", lambda _authorization: None) + terminal_snapshot_receipts["sb-vm"] = {"source_id": "sb-vm", "image_id": None} + manager = SimpleNamespace(take_snapshot=AsyncMock(), stop_sandbox=AsyncMock()) + monkeypatch.setattr("src.sandbox.manager.SandboxManager", lambda: manager) + with pytest.raises(web_api.HTTPException): + await _call_generic_snapshot({"sandbox_id": "sb-vm", "sandbox_backend": "modal-vm"}) + manager.take_snapshot.assert_not_awaited() + manager.stop_sandbox.assert_not_awaited() + + def _patch_dependencies(monkeypatch: pytest.MonkeyPatch): monkeypatch.setattr(web_api, "require_auth", lambda _authorization: None) monkeypatch.setattr(web_api, "validate_control_plane_url", lambda _url: True) @@ -705,7 +855,7 @@ async def test_generic_snapshot_rejects_expired_deadline_without_provider_call(m assert exc.value.status_code == 408 assert exc.value.detail == "snapshot deadline expired" - get_sandbox_by_id.assert_awaited_once_with("modal-session-1") + get_sandbox_by_id.assert_not_awaited() snapshot_filesystem.aio.assert_not_awaited() @@ -723,7 +873,7 @@ async def test_generic_snapshot_rejects_boolean_deadline_without_provider_call( assert exc.value.status_code == 400 assert exc.value.detail == "deadline_at_ms must be a number" - get_sandbox_by_id.assert_awaited_once_with("modal-session-1") + get_sandbox_by_id.assert_not_awaited() snapshot_filesystem.aio.assert_not_awaited() @@ -751,20 +901,52 @@ async def test_vm_snapshot_retirement_shares_the_request_deadline(monkeypatch): monkeypatch.setattr(web_api, "require_auth", lambda _authorization: None) monkeypatch.setattr(web_api, "time", SimpleNamespace(time=lambda: 1000)) - async def delayed_termination(**kwargs): - assert kwargs == {"wait": True} + async def delayed_termination(_id): + assert _id == "sb-vm" await asyncio.sleep(10) terminate = SimpleNamespace(aio=AsyncMock(side_effect=delayed_termination)) handle = SimpleNamespace( - sandbox_backend="modal-vm", modal_sandbox=SimpleNamespace(terminate=terminate) + sandbox_backend="modal-vm", + modal_object_id="sb-vm", + modal_sandbox=SimpleNamespace(terminate=terminate), ) manager = SimpleNamespace( get_sandbox_by_id=AsyncMock(return_value=handle), take_snapshot=AsyncMock(return_value="im-vm"), + stop_sandbox=AsyncMock(side_effect=delayed_termination), ) monkeypatch.setattr("src.sandbox.manager.SandboxManager", lambda: manager) with pytest.raises(web_api.HTTPException) as exc: - await _call_generic_snapshot({"sandbox_id": "sb-vm", "deadline_at_ms": 1_000_020}) + await _call_generic_snapshot( + {"sandbox_id": "sb-vm", "sandbox_backend": "modal-vm", "deadline_at_ms": 1_000_020} + ) assert exc.value.status_code == 408 - terminate.aio.assert_awaited_once_with(wait=True) + manager.stop_sandbox.assert_awaited_once_with("sb-vm") + + +@pytest.mark.asyncio +async def test_vm_snapshot_preserves_image_when_source_disappears_during_retirement(monkeypatch): + from modal.exception import NotFoundError + + monkeypatch.setattr(web_api, "require_auth", lambda _authorization: None) + terminate = SimpleNamespace(aio=AsyncMock(side_effect=NotFoundError("already gone"))) + handle = SimpleNamespace( + sandbox_backend="modal-vm", + modal_object_id="sb-vm", + modal_sandbox=SimpleNamespace(terminate=terminate), + ) + manager = SimpleNamespace( + get_sandbox_by_id=AsyncMock(return_value=handle), + take_snapshot=AsyncMock(return_value="im-recoverable"), + stop_sandbox=SandboxManager().stop_sandbox, + ) + monkeypatch.setattr("src.sandbox.manager.SandboxManager", lambda: manager) + from_id = SimpleNamespace(aio=AsyncMock(return_value=handle.modal_sandbox)) + monkeypatch.setattr("src.sandbox.manager.modal.Sandbox.from_id", from_id) + result = await _call_generic_snapshot({"sandbox_id": "sb-vm", "sandbox_backend": "modal-vm"}) + assert result["data"] == { + "source_stopped": True, + "image_id": "im-recoverable", + "sandbox_id": "sb-vm", + } diff --git a/packages/sandbox-runtime/src/sandbox_runtime/docker_service.py b/packages/sandbox-runtime/src/sandbox_runtime/docker_service.py index 2a3c8277e4..c603f6e789 100644 --- a/packages/sandbox-runtime/src/sandbox_runtime/docker_service.py +++ b/packages/sandbox-runtime/src/sandbox_runtime/docker_service.py @@ -51,18 +51,20 @@ def __init__( self.stop_timeout_seconds = stop_timeout_seconds self.log_path = log_path self._process: asyncio.subprocess.Process | None = None - self._stopping = False + self._exit_expected = False + self._preparation_finished = asyncio.Event() + self._preparation_finished.set() @property - def stopping(self) -> bool: - """Whether the last exit was requested, as opposed to an unexpected daemon death.""" - return self._stopping + def exit_expected(self) -> bool: + """Whether the exit is confirmed preparation or deliberate supervisor teardown.""" + return self._exit_expected async def start(self) -> None: """Start the daemon and wait, under a deadline, until ``docker info`` succeeds.""" if self._process is not None: raise RuntimeError("Docker service already started") - self._stopping = False + self._exit_expected = False self._process = await self._spawn_daemon() try: async with asyncio.timeout(self.start_timeout_seconds): @@ -138,7 +140,9 @@ async def wait(self) -> int: process = self._process if process is None: raise RuntimeError("Docker service is not running") - return await wait_for_process_exit(process) + exit_code = await wait_for_process_exit(process) + await self._preparation_finished.wait() + return exit_code async def prepare_for_snapshot(self) -> None: """Stop the daemon cleanly so an image build can be snapshotted. @@ -151,31 +155,38 @@ async def prepare_for_snapshot(self) -> None: process = self._process if process is None or process.returncode is not None: raise RuntimeError("Required Docker daemon exited before build preparation") - self._stopping = True - process.send_signal(signal.SIGTERM) + self._preparation_finished.clear() try: - async with asyncio.timeout(self.stop_timeout_seconds): - if await wait_for_process_exit(process) != 0: - raise RuntimeError("Docker build preparation did not stop cleanly") - except TimeoutError: - await self.stop() - raise RuntimeError( - "Docker build preparation exceeded its clean shutdown deadline" - ) from None - # A clean daemon exit means it already stopped containerd and BuildKit; - # anything still alive in the group is a straggler, not a dependency. - with contextlib.suppress(ProcessLookupError): - os.killpg(process.pid, signal.SIGKILL) - self._process = None - # A reusable image must never include secret-bearing daemon diagnostics. - Path(self.log_path).write_bytes(b"") - self.log.info("docker.prepared") + process.send_signal(signal.SIGTERM) + try: + async with asyncio.timeout(self.stop_timeout_seconds): + if await wait_for_process_exit(process) != 0: + raise RuntimeError("Docker build preparation did not stop cleanly") + except TimeoutError: + await self.stop() + raise RuntimeError( + "Docker build preparation exceeded its clean shutdown deadline" + ) from None + # A clean daemon exit means it already stopped containerd and BuildKit; + # anything still alive in the group is a straggler, not a dependency. + with contextlib.suppress(ProcessLookupError): + os.killpg(process.pid, signal.SIGKILL) + self._process = None + # A reusable image must never include secret-bearing daemon diagnostics. + Path(self.log_path).write_bytes(b"") + self.log.info("docker.prepared") + self._exit_expected = True + except BaseException: + self._exit_expected = False + raise + finally: + self._preparation_finished.set() async def stop(self) -> None: """Bounded graceful termination, then reap every owned process.""" process = self._process self._process = None - self._stopping = True + self._exit_expected = True if process is None: return diff --git a/packages/sandbox-runtime/src/sandbox_runtime/supervisor.py b/packages/sandbox-runtime/src/sandbox_runtime/supervisor.py index 8a0928cd52..787d9a9286 100644 --- a/packages/sandbox-runtime/src/sandbox_runtime/supervisor.py +++ b/packages/sandbox-runtime/src/sandbox_runtime/supervisor.py @@ -405,7 +405,7 @@ async def _watch_docker(self) -> None: self._docker_watch_failure = error self.shutdown_event.set() return - if service.stopping: + if service.exit_expected: return self.log.error("docker.exited_unexpectedly") self._docker_watch_failure = RuntimeError("Required Docker daemon exited unexpectedly") @@ -720,8 +720,10 @@ async def run(self, repo_image_callback: RepoImageBuildCallback | None = None) - # is not a requested cancellation and the failure must # still reach the control plane. try: - async with asyncio.timeout(FATAL_ERROR_REPORT_TIMEOUT_SECONDS): - await repo_image_callback.report_failure(error_message) + # The callback owns its bounded retries; the daemon failure's + # shutdown signal must not cancel delivery. + if not await repo_image_callback.report_failure(error_message): + self.log.error("image_build.failure_report_failed") except Exception: self.log.error("image_build.failure_report_failed") else: diff --git a/packages/sandbox-runtime/tests/test_docker_service.py b/packages/sandbox-runtime/tests/test_docker_service.py index 91f8ace204..5d3d26df3f 100644 --- a/packages/sandbox-runtime/tests/test_docker_service.py +++ b/packages/sandbox-runtime/tests/test_docker_service.py @@ -123,7 +123,7 @@ async def test_ready_then_clean_preparation_leaves_no_owned_process(processes, t assert daemon.returncode == 0 assert _group_gone(daemon) - assert service.stopping is True + assert service.exit_expected is True await service.stop() assert all(child.returncode is not None for child in processes.children) @@ -168,10 +168,14 @@ async def test_nonzero_daemon_exit_cannot_be_a_prepared_build(processes, tmp_pat processes.daemon_exit = 1 service = _service(tmp_path) await service.start() + watcher = asyncio.create_task(service.wait()) + await asyncio.sleep(0) with pytest.raises(RuntimeError, match="did not stop cleanly"): await service.prepare_for_snapshot() + assert await watcher == 1 + assert service.exit_expected is False await service.stop() assert all(child.returncode is not None for child in processes.children) @@ -183,10 +187,10 @@ async def test_unexpected_exit_is_observable_and_not_a_requested_stop(processes, daemon.kill() assert await service.wait() != 0 - assert service.stopping is False + assert service.exit_expected is False await service.stop() - assert service.stopping is True + assert service.exit_expected is True async def test_daemon_that_ignores_sigterm_is_killed_and_never_a_prepared_build( @@ -200,6 +204,7 @@ async def test_daemon_that_ignores_sigterm_is_killed_and_never_a_prepared_build( with pytest.raises(RuntimeError, match="clean shutdown deadline"): await service.prepare_for_snapshot() + assert service.exit_expected is False assert daemon.returncode is not None assert _group_gone(daemon) diff --git a/packages/sandbox-runtime/tests/test_supervisor_lifecycle.py b/packages/sandbox-runtime/tests/test_supervisor_lifecycle.py index 50db2e99fb..ea36cf867c 100644 --- a/packages/sandbox-runtime/tests/test_supervisor_lifecycle.py +++ b/packages/sandbox-runtime/tests/test_supervisor_lifecycle.py @@ -214,7 +214,7 @@ async def test_code_server_restart_exhaustion_is_nonfatal(tmp_path, monkeypatch) def _docker_service(events, *, prepare_error=None): service = MagicMock() - service.stopping = False + service.exit_expected = False exited = asyncio.Event() async def start(): @@ -228,11 +228,11 @@ async def prepare_for_snapshot(): events.append("docker:prepare") if prepare_error is not None: raise prepare_error - service.stopping = True + service.exit_expected = True async def stop(): events.append("docker:stop") - service.stopping = True + service.exit_expected = True exited.set() service.start = AsyncMock(side_effect=start) @@ -348,7 +348,8 @@ async def test_build_preparation_failure_is_reported_as_a_failed_build( assert supervisor.docker_service.stop.await_count == 1 -async def test_daemon_exit_during_build_hooks_fails_the_build(tmp_path, monkeypatch): +@pytest.mark.parametrize("reported", [True, False]) +async def test_daemon_exit_during_build_hooks_fails_the_build(tmp_path, monkeypatch, reported): events = [] supervisor, repository, *_ = _docker_supervisor(tmp_path, events, monkeypatch) monkeypatch.setenv("IMAGE_BUILD_MODE", "true") @@ -356,6 +357,15 @@ async def test_daemon_exit_during_build_hooks_fails_the_build(tmp_path, monkeypa callback = MagicMock() callback.report_success = AsyncMock() callback.report_failure = AsyncMock() + # A zero fatal-report bound must not cancel the separate build callback policy. + monkeypatch.setattr("sandbox_runtime.supervisor.FATAL_ERROR_REPORT_TIMEOUT_SECONDS", 0) + + async def report_failure(_error): + await asyncio.sleep(0) + events.append("failure:reported") + return reported + + callback.report_failure.side_effect = report_failure async def boot(_mode, _ports): events.append("repository:build") @@ -370,6 +380,9 @@ async def boot(_mode, _ports): callback.report_failure.assert_awaited_once() assert "exited unexpectedly" in callback.report_failure.await_args.args[0] supervisor._report_fatal_error.assert_awaited_once() + assert "failure:reported" in events + if not reported: + supervisor.log.error.assert_any_call("image_build.failure_report_failed") async def test_daemon_exit_during_session_is_fatal(tmp_path, monkeypatch): diff --git a/packages/shared/src/types/image-builds.ts b/packages/shared/src/types/image-builds.ts index 54e70f1c75..049f75e0be 100644 --- a/packages/shared/src/types/image-builds.ts +++ b/packages/shared/src/types/image-builds.ts @@ -8,6 +8,19 @@ */ import { z } from "zod"; +import type { SandboxProviderName } from "./integrations"; + +/** Backends that can build and boot prebuilt images. + * Deployment admission (such as Daytona opt-in) is a separate policy. + */ +export const IMAGE_BUILD_PROVIDER_IDS = [ + "modal", + "modal-vm", + "vercel", + "opencomputer", + "e2b", + "daytona", +] as const satisfies readonly SandboxProviderName[]; /** Mirrors the `image_builds.status` column. */ export const imageBuildStatusSchema = z.enum(["building", "ready", "failed", "superseded"]); diff --git a/packages/web/src/lib/sandbox-provider.ts b/packages/web/src/lib/sandbox-provider.ts index c6e786d106..8e26e407b1 100644 --- a/packages/web/src/lib/sandbox-provider.ts +++ b/packages/web/src/lib/sandbox-provider.ts @@ -2,6 +2,7 @@ * Public sandbox backend helpers for the web app. */ +import { IMAGE_BUILD_PROVIDER_IDS } from "@open-inspect/shared/types/image-builds"; import { isSandboxProviderName, supportsConfigurableSandboxResources as providerSupportsConfigurableSandboxResources, @@ -11,26 +12,12 @@ import { export type PublicSandboxProvider = SandboxProviderName; -/** - * Backends that can build and boot prebuilt repo/environment images. Mirrors - * IMAGE_BUILD_PROVIDERS in the control plane's image-builds/provider-policy.ts — - * a provider gains image-build support in both places or neither. - */ -const REPO_IMAGE_PROVIDERS = [ - "modal", - "modal-vm", - "vercel", - "opencomputer", - "e2b", - "daytona", -] as const; - /** * The single 501 body every image-build route answers with when the deployment's * provider has no image support. Derived from the list so adding a provider * cannot leave a stale message behind. */ -export const REPO_IMAGES_UNSUPPORTED_MESSAGE = `Image builds are only available when SANDBOX_PROVIDER=${formatProviderList(REPO_IMAGE_PROVIDERS)}`; +export const REPO_IMAGES_UNSUPPORTED_MESSAGE = `Image builds are only available when SANDBOX_PROVIDER=${formatProviderList(IMAGE_BUILD_PROVIDER_IDS)}`; export function getPublicSandboxProvider(): PublicSandboxProvider { const rawValue = process.env.NEXT_PUBLIC_SANDBOX_PROVIDER ?? process.env.SANDBOX_PROVIDER; @@ -47,7 +34,7 @@ export function getPublicSandboxProvider(): PublicSandboxProvider { } export function supportsRepoImages(): boolean { - return (REPO_IMAGE_PROVIDERS as readonly string[]).includes(getPublicSandboxProvider()); + return (IMAGE_BUILD_PROVIDER_IDS as readonly string[]).includes(getPublicSandboxProvider()); } export function supportsConfigurableSandboxResources(): boolean { @@ -60,7 +47,7 @@ export function supportsConfigurableSandboxTimeout(): boolean { /** The providers named in the unsupported-provider copy, in display order. */ export function getRepoImageProviders(): readonly PublicSandboxProvider[] { - return REPO_IMAGE_PROVIDERS; + return IMAGE_BUILD_PROVIDER_IDS; } function isPublicSandboxProvider(value: string): value is PublicSandboxProvider { diff --git a/terraform/environments/production/tests/modal_backends.tftest.hcl b/terraform/environments/production/tests/modal_backends.tftest.hcl index 0093ecfb37..dcbe1a1fed 100644 --- a/terraform/environments/production/tests/modal_backends.tftest.hcl +++ b/terraform/environments/production/tests/modal_backends.tftest.hcl @@ -50,6 +50,10 @@ run "gvisor_uses_shared_modal_infrastructure" { condition = local.use_modal_backend && length(module.modal_app) == 1 && length(data.external.modal_source_hash) == 1 error_message = "Standard Modal must provision its shared infrastructure." } + assert { + condition = !module.modal_app[0].vm_image_build_enabled + error_message = "Standard Modal must not request VM image verification." + } } run "vm_uses_shared_modal_infrastructure" { command = plan @@ -58,6 +62,10 @@ run "vm_uses_shared_modal_infrastructure" { condition = local.use_modal_backend && length(module.modal_app) == 1 && length(data.external.modal_source_hash) == 1 error_message = "Modal VM must provision the same Modal module and credentials." } + assert { + condition = module.modal_app[0].vm_image_build_enabled + error_message = "Modal VM selection must reach the module deployment trigger." + } } run "vm_requires_modal_credentials" { command = plan diff --git a/terraform/environments/production/variables.tf b/terraform/environments/production/variables.tf index 7f8b19b3d0..dfc4aee259 100644 --- a/terraform/environments/production/variables.tf +++ b/terraform/environments/production/variables.tf @@ -666,13 +666,13 @@ variable "nextauth_secret" { # ============================================================================= variable "sandbox_provider" { - description = "Sandbox backend for session execution: 'modal', 'daytona', 'vercel', 'opencomputer', or 'e2b'" + description = "Sandbox backend for session execution: 'modal', 'modal-vm', 'daytona', 'vercel', 'opencomputer', or 'e2b'" type = string default = "modal" validation { condition = contains(["modal", "modal-vm", "daytona", "vercel", "opencomputer", "e2b"], var.sandbox_provider) - error_message = "sandbox_provider must be 'modal', 'daytona', 'vercel', 'opencomputer', or 'e2b'." + error_message = "sandbox_provider must be 'modal', 'modal-vm', 'daytona', 'vercel', 'opencomputer', or 'e2b'." } } diff --git a/terraform/modules/modal-app/outputs.tf b/terraform/modules/modal-app/outputs.tf index b0704f6fee..2a88eaad6f 100644 --- a/terraform/modules/modal-app/outputs.tf +++ b/terraform/modules/modal-app/outputs.tf @@ -8,6 +8,11 @@ output "deploy_id" { value = null_resource.modal_deploy.id } +output "vm_image_build_enabled" { + description = "Whether this deployment builds and verifies a new VM image" + value = null_resource.modal_deploy.triggers.build_vm_image == "true" +} + output "api_health_url" { description = "URL of the health check endpoint" value = "https://${local.modal_workspace_slug}--${var.app_name}-api-health.modal.run" From 86d77abef7ee67f9cdb130720e26e785e63e9dca Mon Sep 17 00:00:00 2001 From: Cole Murray Date: Tue, 22 Sep 2026 23:50:57 -0700 Subject: [PATCH 05/22] fix: commit Modal VM captures before source retirement (cherry picked from commit da15bd8be03c866e702c27e9941c0b5adbbf3e6c) --- .../2026-09-22-modal-vm-backend-identities.md | 24 +++--- packages/control-plane/src/sandbox/client.ts | 12 ++- .../alarm-inactivity-effects.test.ts | 2 +- .../src/sandbox/lifecycle/manager.test.ts | 6 +- .../src/sandbox/lifecycle/manager.ts | 7 +- .../control-plane/src/sandbox/provider.ts | 6 +- .../sandbox/providers/modal-backends.test.ts | 67 +++++++++++++-- .../src/sandbox/providers/modal-provider.ts | 26 ++++-- .../sandbox/providers/vercel/provider.test.ts | 2 +- .../src/sandbox/providers/vercel/provider.ts | 2 +- .../session/sandbox-shutdown-repository.ts | 1 + .../src/session/sandbox-shutdown.test.ts | 84 +++++++++++++++++- .../src/session/sandbox-shutdown.ts | 25 ++++-- .../test/integration/sandbox-shutdown.test.ts | 4 +- packages/modal-infra/src/web_api.py | 86 ++++++++++++++++--- .../tests/test_web_api_build_sandbox.py | 74 ++++++++++++++++ .../tests/test_docker_control.py | 2 + 17 files changed, 364 insertions(+), 66 deletions(-) diff --git a/docs/plans/2026-09-22-modal-vm-backend-identities.md b/docs/plans/2026-09-22-modal-vm-backend-identities.md index cb8ffc1a54..6d5fbd8fa6 100644 --- a/docs/plans/2026-09-22-modal-vm-backend-identities.md +++ b/docs/plans/2026-09-22-modal-vm-backend-identities.md @@ -340,23 +340,25 @@ correctness. Reconcile every outstanding PR finding against the final head. The are preserved regressions; build recovery, probe isolation, and log hygiene are confirmed shipping requirements in this change. Live Docker pause/resume is explicitly deferred. -VM session snapshots use the existing **destructive snapshot** contract: the supervisor quiesces -Docker over a local control socket, the provider captures the filesystem, then waits for VM -retirement. Only then does the response confirm `sourceStopped`; subsequent work restores into a new -generation. Standard Modal snapshots are unchanged. This avoids a new general snapshot protocol. +VM session checkpoints remain **terminal**, but capture and retirement are separate operations. The +supervisor quiesces Docker over a local control socket, Modal captures the filesystem while the +source remains alive, and the control plane commits the image ID before it requests and confirms VM +retirement. Subsequent work restores into a new generation. Standard Modal checkpoints remain +nonterminal. This reuses the existing control-plane receipt-before-retirement ordering. Review hardening preserves those boundaries: - VM session generations share a provider-enforced allocation name, with exact generation ownership tags. A pending provider reference is stored before create/restore so snapshot and stop can resolve an allocation whose HTTP response was lost; this reference is not startup confirmation. -- A terminal VM capture uses that generation's stable source reference as its operation key. Modal - stores its image/source receipt before retirement, and a retry can retrieve it without the source - still existing. After timeout or restart, the lifecycle uses a dedicated read-only receipt lookup, - commits any recovered image, then confirms retirement using the recorded immutable source ID. No - unknown capture is reissued. Modal Dict receipts expire after seven days without access. An - incomplete capture intent stays unknown; this does not guarantee recovery if the underlying - snapshot SDK response is itself lost before the receipt is recorded. +- New VM captures use a distinct Modal endpoint that never retires the source or writes a Modal Dict + receipt. It returns the immutable source ID, which the control plane persists alongside the image + receipt and uses for retirement without a Dict lookup. A lost response leaves the source in place + and the control plane holds the ambiguous outcome; a repeated capture can safely re-use Docker's + idempotent preparation command. A capture without an acknowledged image ID never authorizes + retirement. The prior terminal endpoint and read-only receipt recovery remain temporarily for + in-flight canary captures during the rollout, then can be removed after those operations have + settled. - Rejected allocations are durably fenced before awaited cleanup. Their explicit cleanup marker rearms retirement retries after restart without changing unrelated snapshot/recovery holds. - Switching back to gVisor carries forward the currently deployed verified VM image through a diff --git a/packages/control-plane/src/sandbox/client.ts b/packages/control-plane/src/sandbox/client.ts index e256c353b0..f881f27cd2 100644 --- a/packages/control-plane/src/sandbox/client.ts +++ b/packages/control-plane/src/sandbox/client.ts @@ -71,6 +71,7 @@ const snapshotSandboxModalResponseSchema = z.object({ data: z.object({ image_id: z.string().min(1), source_stopped: z.boolean().optional(), + source_id: z.string().min(1).optional(), }), }); @@ -246,6 +247,7 @@ export interface StopSandboxRequest { export interface SnapshotSandboxResponse { sourceStopped?: boolean; + sourceObjectId?: string; imageId: string; } @@ -320,6 +322,7 @@ export class ModalApiError extends Error { export class ModalClient { private createSandboxUrl: string; private snapshotSandboxUrl: string; + private snapshotVmSandboxUrl: string; private recoverSandboxSnapshotUrl: string; private snapshotBuildSandboxUrl: string; private restoreSandboxUrl: string; @@ -369,6 +372,7 @@ export class ModalClient { modalEndpointUrl(functionName, workspace, environmentWebSuffix, apiUrl); this.createSandboxUrl = url("api-create-sandbox"); this.snapshotSandboxUrl = url("api-snapshot-sandbox"); + this.snapshotVmSandboxUrl = url("api-snapshot-vm-sandbox"); this.recoverSandboxSnapshotUrl = url("api-recover-sandbox-snapshot"); this.snapshotBuildSandboxUrl = url("api-snapshot-build-sandbox"); this.restoreSandboxUrl = url("api-restore-sandbox"); @@ -578,7 +582,7 @@ export class ModalClient { try { const result = await this.postJson( - this.snapshotSandboxUrl, + request.sandboxBackend === "modal-vm" ? this.snapshotVmSandboxUrl : this.snapshotSandboxUrl, endpoint, request.deadlineAtMs === undefined ? MODAL_SNAPSHOT_REQUEST_DEADLINE_MS @@ -597,7 +601,11 @@ export class ModalClient { (status) => (httpStatus = status) ); outcome = "success"; - return { imageId: result.data.image_id, sourceStopped: result.data.source_stopped }; + return { + imageId: result.data.image_id, + sourceStopped: result.data.source_stopped, + sourceObjectId: result.data.source_id, + }; } finally { log.info("modal.request", { event: "modal.request", diff --git a/packages/control-plane/src/sandbox/lifecycle/alarm-inactivity-effects.test.ts b/packages/control-plane/src/sandbox/lifecycle/alarm-inactivity-effects.test.ts index ab4c6edbba..15f5b79d19 100644 --- a/packages/control-plane/src/sandbox/lifecycle/alarm-inactivity-effects.test.ts +++ b/packages/control-plane/src/sandbox/lifecycle/alarm-inactivity-effects.test.ts @@ -140,7 +140,7 @@ describe("inactivity alarm effects", () => { sandbox, createMockProvider({ capabilities: { - snapshotStopsSandbox: true, + snapshotRequiresShutdown: true, supportsExplicitStop: true, supportsPersistentResume: false, }, diff --git a/packages/control-plane/src/sandbox/lifecycle/manager.test.ts b/packages/control-plane/src/sandbox/lifecycle/manager.test.ts index 46f8abf2fe..c338896e36 100644 --- a/packages/control-plane/src/sandbox/lifecycle/manager.test.ts +++ b/packages/control-plane/src/sandbox/lifecycle/manager.test.ts @@ -385,7 +385,7 @@ describe("final graceful shutdown lifecycle integration", () => { }); it("routes destructive ordinary snapshots through confirmed graceful shutdown", async () => { - const f = fixture(createMockProvider({ capabilities: { snapshotStopsSandbox: true } })); + const f = fixture(createMockProvider({ capabilities: { snapshotRequiresShutdown: true } })); await f.manager.triggerSnapshot("execution_complete"); expect(f.shutdown.requestShutdown).toHaveBeenCalledWith("execution_complete"); expect(f.provider.takeSnapshot).not.toHaveBeenCalled(); @@ -394,7 +394,7 @@ describe("final graceful shutdown lifecycle integration", () => { it("does not fall through to a destructive checkpoint when shutdown declines it", async () => { const sandbox = createMockSandbox({ status: "ready" }); const f = fixture( - createMockProvider({ capabilities: { snapshotStopsSandbox: true } }), + createMockProvider({ capabilities: { snapshotRequiresShutdown: true } }), sandbox ); f.shutdown.requestShutdown.mockResolvedValue("held"); @@ -2663,7 +2663,7 @@ describe("SandboxLifecycleManager", () => { const storage = createMockStorage(createMockSession(), sandbox); const broadcaster = createMockBroadcaster(); const provider = createMockProvider({ - capabilities: { snapshotStopsSandbox: true }, + capabilities: { snapshotRequiresShutdown: true }, takeSnapshot: vi.fn(async () => ({ success: false, error: "capture failed" })), }); const shutdown = createCheckpointShutdown(provider, storage, broadcaster); diff --git a/packages/control-plane/src/sandbox/lifecycle/manager.ts b/packages/control-plane/src/sandbox/lifecycle/manager.ts index 7fcebe8c8e..46bd4185de 100644 --- a/packages/control-plane/src/sandbox/lifecycle/manager.ts +++ b/packages/control-plane/src/sandbox/lifecycle/manager.ts @@ -1431,9 +1431,10 @@ export class SandboxLifecycleManager */ async triggerSnapshot(reason: string): Promise { if (this.shutdown.isHolding()) return; - // A Vercel snapshot stops the source. It requires the same preparation - // and replacement ordering as a final snapshot, even after a prompt. - if (this.provider.capabilities.snapshotStopsSandbox) { + // Some providers require terminal shutdown before capturing an ordinary + // checkpoint. The source may be retired by the provider or after the + // control plane commits its capture receipt. + if (this.provider.capabilities.snapshotRequiresShutdown) { const ownership = await this.shutdown.requestShutdown(reason); if (ownership !== "unmanaged") return; } diff --git a/packages/control-plane/src/sandbox/provider.ts b/packages/control-plane/src/sandbox/provider.ts index 1916fccf8c..7c076a48f4 100644 --- a/packages/control-plane/src/sandbox/provider.ts +++ b/packages/control-plane/src/sandbox/provider.ts @@ -62,8 +62,8 @@ export interface SandboxProviderCapabilities { supportsPersistentResume?: boolean; /** Whether the provider can stop a sandbox explicitly via API */ supportsExplicitStop?: boolean; - /** Whether taking a snapshot also stops the source sandbox. */ - snapshotStopsSandbox?: boolean; + /** An ordinary checkpoint must enter the terminal shutdown flow first. */ + snapshotRequiresShutdown?: boolean; } export type SandboxLifetime = @@ -307,6 +307,8 @@ export interface SnapshotResult { error?: string; /** True when snapshot creation itself stopped the source sandbox. */ sourceStopped?: boolean; + /** Immutable source ID to retire after committing the snapshot receipt. */ + sourceObjectId?: string; } /** diff --git a/packages/control-plane/src/sandbox/providers/modal-backends.test.ts b/packages/control-plane/src/sandbox/providers/modal-backends.test.ts index 8c004e3b4c..3488c2b4c4 100644 --- a/packages/control-plane/src/sandbox/providers/modal-backends.test.ts +++ b/packages/control-plane/src/sandbox/providers/modal-backends.test.ts @@ -44,7 +44,9 @@ function fixture(confirmation: unknown) { .fn() .mockResolvedValue({ providerSessionId: "sb-1", sandboxBackend: confirmation }), startImageBuildSandbox: vi.fn().mockResolvedValue(undefined), - snapshotSandbox: vi.fn().mockResolvedValue({ imageId: "im-1", sourceStopped: true }), + snapshotSandbox: vi + .fn() + .mockResolvedValue({ imageId: "im-1", sourceStopped: false, sourceObjectId: "sb-1" }), }; return { client, @@ -81,7 +83,7 @@ describe("distinct Modal backend identities", () => { }); expect(fetchMock).toHaveBeenCalledOnce(); }); - it("recovers a lost terminal snapshot response using the same source reference and deadline", async () => { + it("retries a lost VM capture response while retaining the source", async () => { const { client, provider } = fixture("modal-vm"); client.snapshotSandbox.mockRejectedValueOnce(new Error("response lost")); await expect( @@ -91,7 +93,7 @@ describe("distinct Modal backend identities", () => { reason: "shutdown", deadlineAtMs: Date.now() + 60_000, }) - ).resolves.toMatchObject({ success: true, imageId: "im-1", sourceStopped: true }); + ).resolves.toMatchObject({ success: true, imageId: "im-1", sourceStopped: false }); expect(client.snapshotSandbox).toHaveBeenCalledTimes(2); expect(client.snapshotSandbox.mock.calls[0]).toEqual(client.snapshotSandbox.mock.calls[1]); }); @@ -115,7 +117,7 @@ describe("distinct Modal backend identities", () => { ); } expect(provider.name).toBe("modal-vm"); - expect(provider.capabilities.snapshotStopsSandbox).toBe(true); + expect(provider.capabilities.snapshotRequiresShutdown).toBe(true); }); it.each([undefined, null, false, "modal", "future-backend", { unexpected: true }])( @@ -213,15 +215,66 @@ describe("distinct Modal backend identities", () => { } }); - it("requires explicit retirement confirmation for VM session captures", async () => { + it("leaves VM retirement to the control plane after capture", async () => { const { provider, client } = fixture("modal-vm"); const input = { providerObjectId: "sb-1", sessionId: "session-1", reason: "checkpoint" }; await expect(provider.takeSnapshot(input)).resolves.toMatchObject({ success: true, - sourceStopped: true, + sourceStopped: false, + sourceObjectId: "sb-1", }); + expect(client.stopSandbox).not.toHaveBeenCalled(); + }); + + it("uses a separate VM capture endpoint so older deployments cannot stop the source", async () => { + const fetchMock = vi.fn().mockImplementation(async () => + Response.json({ + success: true, + data: { image_id: "im-1", source_stopped: false, source_id: "sb-1", sandbox_id: "sb-1" }, + }) + ); + vi.stubGlobal("fetch", fetchMock); + const client = createModalClient("secret", "acme"); + await client.snapshotSandbox({ + providerObjectId: "sb-1", + sessionId: "session-1", + sandboxBackend: "modal-vm", + }); + await client.snapshotSandbox({ + providerObjectId: "sb-2", + sessionId: "session-2", + sandboxBackend: "modal", + }); + expect(fetchMock.mock.calls[0][0]).toContain("api-snapshot-vm-sandbox"); + expect(fetchMock.mock.calls[1][0]).toContain("api-snapshot-sandbox"); + }); + + it("holds a VM capture without an immutable source ID", async () => { + const { provider, client } = fixture("modal-vm"); client.snapshotSandbox.mockResolvedValue({ imageId: "im-1", sourceStopped: false }); - await expect(provider.takeSnapshot(input)).rejects.toThrow("did not confirm source retirement"); + await expect( + provider.takeSnapshot({ + providerObjectId: "pending-ref", + sessionId: "session-1", + reason: "checkpoint", + }) + ).rejects.toThrow("did not confirm its source ID"); + }); + + it("holds a VM capture that does not confirm source retention", async () => { + const { provider, client } = fixture("modal-vm"); + client.snapshotSandbox.mockResolvedValue({ + imageId: "im-1", + sourceStopped: true, + sourceObjectId: "sb-1", + }); + await expect( + provider.takeSnapshot({ + providerObjectId: "sb-1", + sessionId: "session-1", + reason: "checkpoint", + }) + ).rejects.toThrow("did not confirm source retention"); }); it.each(["modal", "modal-vm"])("keeps dashboard links for %s", (backend) => { diff --git a/packages/control-plane/src/sandbox/providers/modal-provider.ts b/packages/control-plane/src/sandbox/providers/modal-provider.ts index a5514471c1..f40c1cf839 100644 --- a/packages/control-plane/src/sandbox/providers/modal-provider.ts +++ b/packages/control-plane/src/sandbox/providers/modal-provider.ts @@ -114,7 +114,7 @@ export class ModalSandboxProvider implements SandboxProvider, ModalImageBuildPro this.capabilities = { supportsSandboxTimeout: supportsConfigurableSandboxTimeout(this.name), supportsSnapshots: true, - snapshotStopsSandbox: backend === "modal-vm", + snapshotRequiresShutdown: backend === "modal-vm", supportsRestore: true, supportsPersistentResume: false, supportsExplicitStop: true, @@ -263,8 +263,9 @@ export class ModalSandboxProvider implements SandboxProvider, ModalImageBuildPro try { result = await this.client.snapshotSandbox(request, config.correlation); } catch (error) { - // A VM's terminal capture is keyed by this stable source reference. - // One retry can recover the durable receipt after a lost response. + // The VM capture endpoint leaves the source alive until the control + // plane commits the image. Docker preparation is idempotent, so a + // lost response can safely retry the capture. if ( this.name !== "modal-vm" || request.signal?.aborted || @@ -274,16 +275,23 @@ export class ModalSandboxProvider implements SandboxProvider, ModalImageBuildPro result = await this.client.snapshotSandbox(request, config.correlation); } - if (this.name === "modal-vm" && result.sourceStopped !== true) { - throw new SandboxProviderError( - "Modal VM snapshot did not confirm source retirement", - "permanent" - ); + if (this.name === "modal-vm") { + if (result.sourceStopped !== false) + throw new SandboxProviderError( + "Modal VM capture did not confirm source retention", + "permanent" + ); + if (!result.sourceObjectId) + throw new SandboxProviderError( + "Modal VM capture did not confirm its source ID", + "permanent" + ); } return { success: true, imageId: result.imageId, - ...(this.name === "modal-vm" ? { sourceStopped: true } : {}), + sourceStopped: result.sourceStopped === true, + sourceObjectId: result.sourceObjectId, }; } catch (error) { if (error instanceof ModalApiError) { diff --git a/packages/control-plane/src/sandbox/providers/vercel/provider.test.ts b/packages/control-plane/src/sandbox/providers/vercel/provider.test.ts index 07600442b3..c39b9f3d7e 100644 --- a/packages/control-plane/src/sandbox/providers/vercel/provider.test.ts +++ b/packages/control-plane/src/sandbox/providers/vercel/provider.test.ts @@ -173,7 +173,7 @@ describe("VercelSandboxProvider", () => { supportsRestore: true, supportsPersistentResume: false, supportsExplicitStop: true, - snapshotStopsSandbox: true, + snapshotRequiresShutdown: true, }); }); diff --git a/packages/control-plane/src/sandbox/providers/vercel/provider.ts b/packages/control-plane/src/sandbox/providers/vercel/provider.ts index 3f97da1ba7..9d7fe73940 100644 --- a/packages/control-plane/src/sandbox/providers/vercel/provider.ts +++ b/packages/control-plane/src/sandbox/providers/vercel/provider.ts @@ -92,7 +92,7 @@ export class VercelSandboxProvider implements SandboxProvider { supportsRestore: true, supportsPersistentResume: false, supportsExplicitStop: true, - snapshotStopsSandbox: true, + snapshotRequiresShutdown: true, }; constructor( diff --git a/packages/control-plane/src/session/sandbox-shutdown-repository.ts b/packages/control-plane/src/session/sandbox-shutdown-repository.ts index 5594feea2a..b219195e25 100644 --- a/packages/control-plane/src/session/sandbox-shutdown-repository.ts +++ b/packages/control-plane/src/session/sandbox-shutdown-repository.ts @@ -7,6 +7,7 @@ import { SessionStorageIntegrityError } from "./types"; const receiptSchema = z.object({ kind: z.enum(["snapshot", "retained"]), artifactId: z.string().min(1), + sourceObjectId: z.string().min(1).optional(), provider: z.string(), savedAtMs: z.number(), runtimeVersion: z.string().nullable(), diff --git a/packages/control-plane/src/session/sandbox-shutdown.test.ts b/packages/control-plane/src/session/sandbox-shutdown.test.ts index 7bdf4e9a73..aaa4fa620a 100644 --- a/packages/control-plane/src/session/sandbox-shutdown.test.ts +++ b/packages/control-plane/src/session/sandbox-shutdown.test.ts @@ -158,16 +158,16 @@ function preparedEvent( describe("SandboxShutdownCoordinator", () => { beforeEach(() => vi.restoreAllMocks()); - it("recovers a terminal capture receipt after response loss and reconstruction without recapturing", async () => { + it("recovers a legacy terminal capture receipt after response loss", async () => { const takeSnapshot = vi.fn(async () => { throw new Error("response aborted after capture and retirement"); }); - const recoverSnapshotReceipt = vi.fn().mockRejectedValueOnce(new Error("network offline")); + const recoverSnapshotReceipt = vi.fn(); const stopSandbox = vi.fn(async () => ({ success: true as const })); const f = fixture( provider({ name: "modal-vm", - capabilities: { ...provider().capabilities, snapshotStopsSandbox: true }, + capabilities: { ...provider().capabilities, snapshotRequiresShutdown: true }, takeSnapshot, recoverSnapshotReceipt, stopSandbox, @@ -177,9 +177,12 @@ describe("SandboxShutdownCoordinator", () => { await f.shutdown.requestShutdown("checkpoint"); f.shutdown.prepared(preparedEvent(f.store.value!)); await f.shutdown.handleAlarm(); - expect(f.store.value).toMatchObject({ phase: "unknown", captureReceiptPending: true }); + expect(f.store.value).toMatchObject({ phase: "unknown", captureReceiptPending: false }); expect(stopSandbox).not.toHaveBeenCalled(); + // A persisted row from the previous VM endpoint may still be pending when + // the control plane rolls forward. Only that legacy row reads Modal Dict. + f.store.write({ ...f.store.value!, captureReceiptPending: true }); recoverSnapshotReceipt.mockResolvedValue({ imageId: "newest-image" }); f.setNow(f.store.value!.captureByMs! + 60_000); // Original request deadline is gone. const restarted = new SandboxShutdownCoordinator(f.deps as never); @@ -203,6 +206,79 @@ describe("SandboxShutdownCoordinator", () => { ); }); + it("commits a VM image before retiring its retained source", async () => { + const takeSnapshot = vi.fn(async () => ({ + success: true as const, + imageId: "vm-image", + sourceStopped: false, + sourceObjectId: "sb-immutable", + })); + const stopSandbox = vi.fn(async () => ({ success: true as const })); + const recoverSnapshotReceipt = vi.fn(); + const f = fixture( + provider({ + name: "modal-vm", + capabilities: { ...provider().capabilities, snapshotRequiresShutdown: true }, + takeSnapshot, + stopSandbox, + recoverSnapshotReceipt, + }) + ); + await readyFinite(f); + await f.shutdown.requestShutdown("execution_complete"); + f.shutdown.prepared(preparedEvent(f.store.value!)); + await f.shutdown.handleAlarm(); + + expect(f.store.value).toMatchObject({ + phase: "saved", + sourceRetired: true, + captureReceiptPending: false, + receipt: { + artifactId: "vm-image", + provider: "modal-vm", + sourceObjectId: "sb-immutable", + }, + }); + expect(f.deps.sandbox.recordSandboxSnapshot).toHaveBeenCalledWith( + GENERATION.sandboxId, + "vm-image", + "runtime-1" + ); + expect(f.deps.sandbox.recordSandboxSnapshot.mock.invocationCallOrder[0]).toBeLessThan( + stopSandbox.mock.invocationCallOrder[0] + ); + expect(stopSandbox).toHaveBeenCalledWith( + expect.objectContaining({ providerObjectId: "sb-immutable" }) + ); + expect(recoverSnapshotReceipt).not.toHaveBeenCalled(); + }); + + it("holds a lost VM capture response without retiring the source or polling Modal Dict", async () => { + const takeSnapshot = vi.fn(async () => { + throw new Error("capture response lost"); + }); + const stopSandbox = vi.fn(async () => ({ success: true as const })); + const recoverSnapshotReceipt = vi.fn(); + const f = fixture( + provider({ + name: "modal-vm", + capabilities: { ...provider().capabilities, snapshotRequiresShutdown: true }, + takeSnapshot, + stopSandbox, + recoverSnapshotReceipt, + }) + ); + await readyFinite(f); + await f.shutdown.requestShutdown("execution_complete"); + f.shutdown.prepared(preparedEvent(f.store.value!)); + await f.shutdown.handleAlarm(); + await f.shutdown.handleAlarm(); + + expect(f.store.value).toMatchObject({ phase: "unknown", captureReceiptPending: false }); + expect(stopSandbox).not.toHaveBeenCalled(); + expect(recoverSnapshotReceipt).not.toHaveBeenCalled(); + }); + it("does not publish a recovered capture for a replaced generation", async () => { const recoverSnapshotReceipt = vi.fn(async () => { f.sandboxRow.created_at += 1; diff --git a/packages/control-plane/src/session/sandbox-shutdown.ts b/packages/control-plane/src/session/sandbox-shutdown.ts index 813aceb6fc..d92df458ec 100644 --- a/packages/control-plane/src/session/sandbox-shutdown.ts +++ b/packages/control-plane/src/session/sandbox-shutdown.ts @@ -798,8 +798,10 @@ export class SandboxShutdownCoordinator { const capturing = { ...state, phase: "capturing" as const, - captureReceiptPending: - !!provider.capabilities.snapshotStopsSandbox && !!provider.recoverSnapshotReceipt, + // Legacy Modal VM captures could retire the source before returning. + // New captures keep it alive until the receipt is committed here. + // Existing rows with captureReceiptPending=true remain recoverable. + captureReceiptPending: false, }; this.publish(capturing); await this.deps.alarm.schedule(state.captureByMs!); @@ -816,6 +818,7 @@ export class SandboxShutdownCoordinator { }; let artifactId = state.providerObjectId; let sourceStopped = retained; + let sourceObjectId: string | undefined; if (retained) { if (!provider.stopSandbox) throw new Error("Provider cannot preserve-stop this sandbox"); const result = await this.bounded(state.captureByMs!, (signal) => @@ -832,12 +835,14 @@ export class SandboxShutdownCoordinator { ); artifactId = result.imageId; sourceStopped = result.sourceStopped; + sourceObjectId = result.sourceObjectId; } if (!this.owns(capturing)) return; const retiring = this.commitCaptureReceipt( capturing, artifactId, - retained ? "retained" : "snapshot" + retained ? "retained" : "snapshot", + sourceObjectId ); if (sourceStopped) this.finish(retiring); else await this.retire(retiring); @@ -858,11 +863,13 @@ export class SandboxShutdownCoordinator { private commitCaptureReceipt( state: ShutdownRecord, artifactId: string, - kind: "retained" | "snapshot" + kind: "retained" | "snapshot", + sourceObjectId?: string ): ShutdownRecord { const receipt = { kind, artifactId, + ...(sourceObjectId ? { sourceObjectId } : {}), provider: this.deps.provider.name, savedAtMs: this.now(), runtimeVersion: this.deps.sandbox.getSandbox()?.runtime_version ?? null, @@ -950,7 +957,7 @@ export class SandboxShutdownCoordinator { await this.deps.alarm.schedule(deadlineAtMs); const result = await this.bounded(deadlineAtMs, (signal) => this.deps.provider.stopSandbox!({ - providerObjectId: state.providerObjectId!, + providerObjectId: state.receipt!.sourceObjectId ?? state.providerObjectId!, sessionId: session.session_name || session.id, reason: state.reason!, intent: state.receipt!.kind === "snapshot" ? "destroy" : "preserve", @@ -1063,7 +1070,7 @@ export class SandboxShutdownCoordinator { sessionId: string, reason: string, deadlineAtMs: number - ): Promise<{ imageId: string; sourceStopped: boolean }> { + ): Promise<{ imageId: string; sourceStopped: boolean; sourceObjectId?: string }> { if (!this.deps.provider.takeSnapshot) throw new Error("Provider has no snapshot operation"); const result = await this.bounded(deadlineAtMs, (signal) => this.deps.provider.takeSnapshot!({ @@ -1076,6 +1083,10 @@ export class SandboxShutdownCoordinator { ); if (!result.success || !result.imageId) throw new Error(result.error ?? "Provider snapshot result is unknown"); - return { imageId: result.imageId, sourceStopped: result.sourceStopped === true }; + return { + imageId: result.imageId, + sourceStopped: result.sourceStopped === true, + sourceObjectId: result.sourceObjectId, + }; } } diff --git a/packages/control-plane/test/integration/sandbox-shutdown.test.ts b/packages/control-plane/test/integration/sandbox-shutdown.test.ts index 1a2681742e..d405ea73ca 100644 --- a/packages/control-plane/test/integration/sandbox-shutdown.test.ts +++ b/packages/control-plane/test/integration/sandbox-shutdown.test.ts @@ -105,7 +105,7 @@ describe("sandbox graceful shutdown wiring", () => { capabilities: { supportsSandboxTimeout: true, supportsSnapshots: true, - snapshotStopsSandbox: true, + snapshotRequiresShutdown: true, supportsRestore: true, supportsExplicitStop: true, }, @@ -540,7 +540,7 @@ describe("sandbox graceful shutdown wiring", () => { supportsRestore: true, supportsExplicitStop: true, supportsPersistentResume: false, - snapshotStopsSandbox: true, + snapshotRequiresShutdown: true, }, createSandbox: async () => { throw new Error("not used by inactivity regression"); diff --git a/packages/modal-infra/src/web_api.py b/packages/modal-infra/src/web_api.py index bd576f48ae..71580ba177 100644 --- a/packages/modal-infra/src/web_api.py +++ b/packages/modal-infra/src/web_api.py @@ -505,6 +505,25 @@ def api_health() -> dict: return {"success": True, "data": {"status": "healthy", "service": "open-inspect-modal"}} +def _snapshot_timeout_seconds(request: dict[str, Any]) -> float: + """Bound a capture by the caller's deadline without extending it.""" + from .sandbox.manager import SNAPSHOT_FILESYSTEM_TIMEOUT_SECONDS + + deadline_at_ms = request.get("deadline_at_ms") + if deadline_at_ms is None: + return SNAPSHOT_FILESYSTEM_TIMEOUT_SECONDS + if ( + isinstance(deadline_at_ms, bool) + or not isinstance(deadline_at_ms, (int, float)) + or not math.isfinite(deadline_at_ms) + ): + raise HTTPException(status_code=400, detail="deadline_at_ms must be a number") + timeout_seconds = (float(deadline_at_ms) / 1000) - time.time() + if timeout_seconds <= 0: + raise HTTPException(status_code=408, detail="snapshot deadline expired") + return timeout_seconds + + @app.function(image=function_image, secrets=[internal_api_secret]) @fastapi_endpoint(method="POST") async def api_snapshot_sandbox( @@ -551,22 +570,11 @@ async def api_snapshot_sandbox( if not sandbox_id: raise HTTPException(status_code=400, detail="sandbox_id is required") - from .sandbox.manager import SNAPSHOT_FILESYSTEM_TIMEOUT_SECONDS, SandboxManager + from .sandbox.manager import SandboxManager manager = SandboxManager() - deadline_at_ms = request.get("deadline_at_ms") - timeout_seconds = SNAPSHOT_FILESYSTEM_TIMEOUT_SECONDS - if deadline_at_ms is not None: - if ( - isinstance(deadline_at_ms, bool) - or not isinstance(deadline_at_ms, (int, float)) - or not math.isfinite(deadline_at_ms) - ): - raise HTTPException(status_code=400, detail="deadline_at_ms must be a number") - timeout_seconds = (deadline_at_ms / 1000) - time.time() - if timeout_seconds <= 0: - raise HTTPException(status_code=408, detail="snapshot deadline expired") + timeout_seconds = _snapshot_timeout_seconds(request) source_stopped = request.get("sandbox_backend") == "modal-vm" try: async with asyncio.timeout(timeout_seconds): @@ -602,6 +610,58 @@ async def api_snapshot_sandbox( } +@app.function(image=function_image, secrets=[internal_api_secret]) +@fastapi_endpoint(method="POST") +async def api_snapshot_vm_sandbox( + request: dict[str, Any], + authorization: str | None = Header(None), + x_trace_id: str | None = Header(None), + x_request_id: str | None = Header(None), + x_session_id: str | None = Header(None), + x_sandbox_id: str | None = Header(None), +) -> dict[str, Any]: + """Capture a prepared VM without retiring it; the control plane owns retirement.""" + async with _execute_endpoint( + endpoint_name="api_snapshot_vm_sandbox", + authorization=authorization, + trace_id=x_trace_id, + request_id=x_request_id, + session_id=x_session_id, + sandbox_id=x_sandbox_id, + ) as execution: + sandbox_id = request.get("sandbox_id") + execution.log_fields["sandbox_id"] = x_sandbox_id or sandbox_id + if not isinstance(sandbox_id, str) or not sandbox_id: + raise HTTPException(status_code=400, detail="sandbox_id is required") + if request.get("sandbox_backend") != "modal-vm": + raise HTTPException(status_code=400, detail="modal-vm backend confirmation is required") + + from .sandbox.manager import SandboxManager + + manager = SandboxManager() + timeout_seconds = _snapshot_timeout_seconds(request) + try: + async with asyncio.timeout(timeout_seconds): + handle = await manager.get_sandbox_by_id(sandbox_id) + if handle is None or handle.sandbox_backend != "modal-vm": + raise HTTPException(status_code=400, detail="Terminal capture requires a VM") + source_id = handle.modal_object_id + if not source_id: + raise HTTPException(status_code=500, detail="VM source ID is unavailable") + image_id = await manager.take_snapshot(handle, timeout_seconds=timeout_seconds) + except (TimeoutError, ModalTimeoutError) as exc: + raise HTTPException(status_code=408, detail="snapshot deadline expired") from exc + return { + "success": True, + "data": { + "source_stopped": False, + "source_id": source_id, + "image_id": image_id, + "sandbox_id": sandbox_id, + }, + } + + @app.function(image=function_image, secrets=[internal_api_secret]) @fastapi_endpoint(method="POST") async def api_recover_sandbox_snapshot( diff --git a/packages/modal-infra/tests/test_web_api_build_sandbox.py b/packages/modal-infra/tests/test_web_api_build_sandbox.py index 4e11bd7a82..f308928bc6 100644 --- a/packages/modal-infra/tests/test_web_api_build_sandbox.py +++ b/packages/modal-infra/tests/test_web_api_build_sandbox.py @@ -59,6 +59,69 @@ async def test_snapshot_receipt_recovery_is_read_only( manager.assert_not_called() +@pytest.mark.asyncio +async def test_vm_capture_leaves_source_alive_until_control_plane_commits( + monkeypatch, terminal_snapshot_receipts +): + monkeypatch.setattr(web_api, "require_auth", lambda _authorization: None) + handle = SimpleNamespace(sandbox_backend="modal-vm", modal_object_id="sb-immutable") + manager = SimpleNamespace( + get_sandbox_by_id=AsyncMock(return_value=handle), + take_snapshot=AsyncMock(side_effect=["im-first", "im-retry"]), + stop_sandbox=AsyncMock(), + ) + monkeypatch.setattr("src.sandbox.manager.SandboxManager", lambda: manager) + request = {"sandbox_id": "sb-vm", "sandbox_backend": "modal-vm"} + + first = await _call_vm_snapshot(request) + second = await _call_vm_snapshot(request) + + assert first["data"] == { + "source_stopped": False, + "source_id": "sb-immutable", + "image_id": "im-first", + "sandbox_id": "sb-vm", + } + assert second["data"]["image_id"] == "im-retry" + assert manager.take_snapshot.await_count == 2 + manager.stop_sandbox.assert_not_awaited() + assert terminal_snapshot_receipts == {} + + +@pytest.mark.asyncio +async def test_vm_capture_rejects_non_vm_source(monkeypatch): + monkeypatch.setattr(web_api, "require_auth", lambda _authorization: None) + manager = SimpleNamespace( + get_sandbox_by_id=AsyncMock(return_value=SimpleNamespace(sandbox_backend="modal")), + take_snapshot=AsyncMock(), + ) + monkeypatch.setattr("src.sandbox.manager.SandboxManager", lambda: manager) + with pytest.raises(web_api.HTTPException, match="Terminal capture requires a VM"): + await _call_vm_snapshot({"sandbox_id": "sb-standard", "sandbox_backend": "modal-vm"}) + manager.take_snapshot.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_vm_capture_failure_never_retires_or_records_a_receipt( + monkeypatch, terminal_snapshot_receipts +): + monkeypatch.setattr(web_api, "require_auth", lambda _authorization: None) + manager = SimpleNamespace( + get_sandbox_by_id=AsyncMock( + return_value=SimpleNamespace( + sandbox_backend="modal-vm", modal_object_id="sb-immutable" + ) + ), + take_snapshot=AsyncMock(side_effect=RuntimeError("capture failed")), + stop_sandbox=AsyncMock(), + ) + monkeypatch.setattr("src.sandbox.manager.SandboxManager", lambda: manager) + with pytest.raises(web_api.HTTPException): + await _call_vm_snapshot({"sandbox_id": "sb-vm", "sandbox_backend": "modal-vm"}) + manager.stop_sandbox.assert_not_awaited() + assert terminal_snapshot_receipts == {} + + @pytest.mark.asyncio @pytest.mark.parametrize("reference", ["sb-vm", 'modal-vm-session:["session","generation"]']) async def test_terminal_capture_uses_real_lookup_identity_and_replays_after_retirement( @@ -209,6 +272,17 @@ async def _call_generic_snapshot(request: dict) -> dict: ) +async def _call_vm_snapshot(request: dict) -> dict: + return await web_api.api_snapshot_vm_sandbox.get_raw_f()( + request, + authorization="Bearer test", + x_trace_id=None, + x_request_id=None, + x_session_id=None, + x_sandbox_id=None, + ) + + async def _call_generic_stop(request: dict) -> dict: return await web_api.api_stop_sandbox.get_raw_f()( request, diff --git a/packages/sandbox-runtime/tests/test_docker_control.py b/packages/sandbox-runtime/tests/test_docker_control.py index 5b2c68fc0e..8dca7ea1f4 100644 --- a/packages/sandbox-runtime/tests/test_docker_control.py +++ b/packages/sandbox-runtime/tests/test_docker_control.py @@ -24,6 +24,8 @@ async def test_preparation_is_acknowledged_only_after_clean_stop(socket_path): with pytest.raises(RuntimeError, match="confirmed shutdown"): await request("status", path) await asyncio.gather(request("prepare", path), request("prepare", path)) + # A lost capture response may prepare the same retained VM again. + await request("prepare", path) service.prepare_for_snapshot.assert_awaited_once() await request("status", path) finally: From f827db3e1462060455a1e309e94b2251aad1e69d Mon Sep 17 00:00:00 2001 From: Cole Murray Date: Tue, 22 Sep 2026 23:52:38 -0700 Subject: [PATCH 06/22] style: format Modal VM capture test (cherry picked from commit 7f66db0e8ffbe23fc571ba77055986804b1db5a3) --- packages/modal-infra/tests/test_web_api_build_sandbox.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/modal-infra/tests/test_web_api_build_sandbox.py b/packages/modal-infra/tests/test_web_api_build_sandbox.py index f308928bc6..51ae8bcec0 100644 --- a/packages/modal-infra/tests/test_web_api_build_sandbox.py +++ b/packages/modal-infra/tests/test_web_api_build_sandbox.py @@ -108,9 +108,7 @@ async def test_vm_capture_failure_never_retires_or_records_a_receipt( monkeypatch.setattr(web_api, "require_auth", lambda _authorization: None) manager = SimpleNamespace( get_sandbox_by_id=AsyncMock( - return_value=SimpleNamespace( - sandbox_backend="modal-vm", modal_object_id="sb-immutable" - ) + return_value=SimpleNamespace(sandbox_backend="modal-vm", modal_object_id="sb-immutable") ), take_snapshot=AsyncMock(side_effect=RuntimeError("capture failed")), stop_sandbox=AsyncMock(), From a168b9b5aa88f1223009ea0d675c5bf0cd3e1be6 Mon Sep 17 00:00:00 2001 From: Cole Murray Date: Wed, 23 Sep 2026 09:43:17 -0700 Subject: [PATCH 07/22] Log Modal VM health and detect stalled heartbeats promptly (cherry picked from commit 0a5679e2486b4740f300054549b3c8741671da38) --- .../sandbox/lifecycle/alarm-policy.test.ts | 15 ++++- .../src/sandbox/lifecycle/alarm-policy.ts | 13 +++- .../src/sandbox/lifecycle/manager.ts | 4 ++ packages/modal-infra/src/sandbox/manager.py | 10 +++ .../src/sandbox_runtime/bridge.py | 43 +++++++++++- .../harness/opencode_stream.py | 1 + .../src/sandbox_runtime/health_snapshot.py | 65 +++++++++++++++++++ .../tests/test_health_snapshot.py | 39 +++++++++++ 8 files changed, 184 insertions(+), 6 deletions(-) create mode 100644 packages/sandbox-runtime/src/sandbox_runtime/health_snapshot.py create mode 100644 packages/sandbox-runtime/tests/test_health_snapshot.py diff --git a/packages/control-plane/src/sandbox/lifecycle/alarm-policy.test.ts b/packages/control-plane/src/sandbox/lifecycle/alarm-policy.test.ts index c7a3857cf5..eade4a4cb3 100644 --- a/packages/control-plane/src/sandbox/lifecycle/alarm-policy.test.ts +++ b/packages/control-plane/src/sandbox/lifecycle/alarm-policy.test.ts @@ -143,7 +143,7 @@ describe("evaluateAlarmPolicy", () => { evaluateAlarmPolicy(row({ status: "ready", last_heartbeat: now - ageMs }), config, now, 0) ).toEqual( offsetMs <= 0 - ? healthy + ? { outcome: "healthy", nextCheckMs: Math.max(1, -offsetMs + 1) } : { outcome: "heartbeat_stale", ageMs, @@ -161,7 +161,7 @@ describe("evaluateAlarmPolicy", () => { it.each<[number | null, number, AlarmFinding]>([ [null, 0, healthy], - [now - 300_000, 0, { outcome: "healthy", nextCheckMs: 300_000 }], + [now - 300_000, 0, { outcome: "healthy", nextCheckMs: 90_001 }], [now - 599_999, 0, healthy], [now - 600_000, 0, { outcome: "inactivity_timeout" }], [now - 600_001, 0, { outcome: "inactivity_timeout" }], @@ -184,6 +184,17 @@ describe("evaluateAlarmPolicy", () => { ).toEqual(expected); }); + it("checks heartbeat before the ten-minute inactivity deadline", () => { + expect( + evaluateAlarmPolicy( + row({ status: "ready", last_heartbeat: now - 30_000, last_activity: now }), + config, + now, + 0 + ) + ).toEqual({ outcome: "healthy", nextCheckMs: 60_001 }); + }); + it.each(["pending", "snapshotting"] as const)("does not stop an idle %s row", (status) => { expect( evaluateAlarmPolicy(row({ status, last_heartbeat: now, last_activity: 0 }), config, now, 0) diff --git a/packages/control-plane/src/sandbox/lifecycle/alarm-policy.ts b/packages/control-plane/src/sandbox/lifecycle/alarm-policy.ts index 05faed8b01..f232d314fb 100644 --- a/packages/control-plane/src/sandbox/lifecycle/alarm-policy.ts +++ b/packages/control-plane/src/sandbox/lifecycle/alarm-policy.ts @@ -82,6 +82,17 @@ export function evaluateAlarmPolicy( case "extend": return { outcome: "inactivity_warning", extensionMs: inactivity.extensionMs }; case "schedule": - return { outcome: "healthy", nextCheckMs: inactivity.nextCheckMs }; + // The inactivity deadline can be ten minutes away. A bridge that stops + // heartbeating just after this alarm must still be noticed at 90s. + return { + outcome: "healthy", + nextCheckMs: + sandbox.last_heartbeat === null + ? inactivity.nextCheckMs + : Math.min( + inactivity.nextCheckMs, + Math.max(1, sandbox.last_heartbeat + config.heartbeat.timeoutMs + 1 - now) + ), + }; } } diff --git a/packages/control-plane/src/sandbox/lifecycle/manager.ts b/packages/control-plane/src/sandbox/lifecycle/manager.ts index 46bd4185de..4101150a67 100644 --- a/packages/control-plane/src/sandbox/lifecycle/manager.ts +++ b/packages/control-plane/src/sandbox/lifecycle/manager.ts @@ -1738,7 +1738,11 @@ export class SandboxLifecycleManager event: "sandbox.heartbeat_stale", last_heartbeat_ms: ageMs, threshold_ms: this.config.heartbeat.timeoutMs, + detection_lag_ms: Math.max(0, ageMs - this.config.heartbeat.timeoutMs), sandbox_status: ctx.sandbox.status, + provider_object_id: ctx.providerObjectId, + connected_clients: ctx.connectedClients, + is_booting: isBooting, }); this.storage.updateSandboxStatus("stale"); // A bridge that connected and then died mid-boot is a boot failure diff --git a/packages/modal-infra/src/sandbox/manager.py b/packages/modal-infra/src/sandbox/manager.py index 887592f61c..fd1ddff955 100644 --- a/packages/modal-infra/src/sandbox/manager.py +++ b/packages/modal-infra/src/sandbox/manager.py @@ -492,6 +492,16 @@ async def _launch_sandbox(self, spec: _SandboxLaunchSpec) -> SandboxHandle: if exposed_ports: create_kwargs["encrypted_ports"] = exposed_ports + log.info( + "sandbox.launch_config", + sandbox_id=sandbox_id, + backend=docker.backend, + cpu_cores=docker.cpu_cores, + memory_mib=docker.memory_mib, + timeout_seconds=config.timeout_seconds, + image_source=type(spec.source).__name__, + ) + repository_image = isinstance(spec.source, _RepositoryImageSource) if docker.enabled: sandbox, adopted = await self._launch_docker_sandbox( diff --git a/packages/sandbox-runtime/src/sandbox_runtime/bridge.py b/packages/sandbox-runtime/src/sandbox_runtime/bridge.py index 80693380d3..6a4a2230ff 100644 --- a/packages/sandbox-runtime/src/sandbox_runtime/bridge.py +++ b/packages/sandbox-runtime/src/sandbox_runtime/bridge.py @@ -65,6 +65,7 @@ build_agent_harness, parse_harness_id, ) +from .health_snapshot import read_health_snapshot from .log_config import configure_logging, get_logger from .prompt_budgets import resolve_prompt_limits from .push_operation import PushOperation, PushRejected, PushRequest @@ -349,6 +350,7 @@ async def run(self) -> None: self.log.warn( "bridge.connect_error", detail=error_str, + **read_health_snapshot(), ) if self.shutdown_event.is_set(): @@ -435,7 +437,13 @@ def _log_disconnect( if connection_fields is None: return log_method = getattr(self.log, level) - log_method("bridge.disconnect", reason=reason, **connection_fields, **fields) + log_method( + "bridge.disconnect", + reason=reason, + **connection_fields, + **fields, + **read_health_snapshot(), + ) def _is_fatal_connection_error(self, error_str: str) -> bool: """Check if a connection error is fatal and shouldn't trigger retry. @@ -565,11 +573,40 @@ def _heartbeat_event(self) -> dict[str, Any]: async def _heartbeat_loop(self) -> None: """Send periodic heartbeat events.""" + heartbeat_count = 0 + last_tick = time.monotonic() + last_delivered: float | None = None while not self.shutdown_event.is_set(): await asyncio.sleep(self.HEARTBEAT_INTERVAL) - + tick = time.monotonic() + tick_delay_ms = max(0, int((tick - last_tick - self.HEARTBEAT_INTERVAL) * 1000)) + last_tick = tick + delivered = False if self.ws and self.ws.state == State.OPEN: - await self._send_event(self._heartbeat_event()) + delivered = await self._send_event(self._heartbeat_event()) + if delivered: + last_delivered = time.monotonic() + + heartbeat_count += 1 + if ( + heartbeat_count == 1 + or heartbeat_count % 4 == 0 + or tick_delay_ms >= 30_000 + or not delivered + ): + self.log.info( + "bridge.health", + heartbeat_delivered=delivered, + heartbeat_tick_delay_ms=tick_delay_ms, + last_heartbeat_delivered_ago_ms=( + int((time.monotonic() - last_delivered) * 1000) + if last_delivered is not None + else None + ), + websocket_open=self.ws is not None and self.ws.state == State.OPEN, + prompt_active=self.activity.current_prompt_task is not None, + **read_health_snapshot(), + ) async def _end_run(self) -> None: """End the run loop from outside it. diff --git a/packages/sandbox-runtime/src/sandbox_runtime/harness/opencode_stream.py b/packages/sandbox-runtime/src/sandbox_runtime/harness/opencode_stream.py index 519ef83f8a..c68bfc874b 100644 --- a/packages/sandbox-runtime/src/sandbox_runtime/harness/opencode_stream.py +++ b/packages/sandbox-runtime/src/sandbox_runtime/harness/opencode_stream.py @@ -524,6 +524,7 @@ def _on_session_error(self, state: _PromptState, props: dict[str, Any]) -> _Stre self._log.error( "bridge.session_error", error_msg=self._extract_error_message(error), + error_name=error.get("name") if isinstance(error, dict) else None, deduped=error_event is None, ) return _StreamStep( diff --git a/packages/sandbox-runtime/src/sandbox_runtime/health_snapshot.py b/packages/sandbox-runtime/src/sandbox_runtime/health_snapshot.py new file mode 100644 index 0000000000..f760a1f28b --- /dev/null +++ b/packages/sandbox-runtime/src/sandbox_runtime/health_snapshot.py @@ -0,0 +1,65 @@ +"""Best-effort Linux resource counters for diagnosing an unresponsive sandbox.""" + +from contextlib import suppress +from pathlib import Path + +_MEMINFO = Path("/proc/meminfo") +_MEMORY_PRESSURE = Path("/proc/pressure/memory") +_CGROUP_MEMORY_CURRENT = Path("/sys/fs/cgroup/memory.current") +_CGROUP_MEMORY_MAX = Path("/sys/fs/cgroup/memory.max") +_CGROUP_MEMORY_EVENTS = Path("/sys/fs/cgroup/memory.events") + + +def _read(path: Path) -> str | None: + try: + return path.read_text() + except OSError: + return None + + +def _mib(value: str) -> int | None: + try: + return int(value) // (1024 * 1024) + except ValueError: + return None + + +def read_health_snapshot() -> dict[str, int | float]: + """Return only available numeric counters; never block heartbeat on missing proc files.""" + result: dict[str, int | float] = {} + + meminfo = _read(_MEMINFO) + if meminfo is not None: + for line in meminfo.splitlines(): + if line.startswith("MemAvailable:"): + with suppress(IndexError, ValueError): + result["memory_available_mib"] = int(line.split()[1]) // 1024 + break + + for path, field in ( + (_CGROUP_MEMORY_CURRENT, "cgroup_memory_current_mib"), + (_CGROUP_MEMORY_MAX, "cgroup_memory_max_mib"), + ): + value = _read(path) + if value is not None and (parsed := _mib(value.strip())) is not None: + result[field] = parsed + + events = _read(_CGROUP_MEMORY_EVENTS) + if events is not None: + for line in events.splitlines(): + key, _, value = line.partition(" ") + if key in {"oom", "oom_kill"}: + with suppress(ValueError): + result[f"cgroup_{key}_count"] = int(value) + + pressure = _read(_MEMORY_PRESSURE) + if pressure is not None: + for line in pressure.splitlines(): + if line.startswith("some "): + for field in line.split()[1:]: + if field.startswith("avg10="): + with suppress(ValueError): + result["memory_psi_some_avg10"] = float(field.removeprefix("avg10=")) + break + + return result diff --git a/packages/sandbox-runtime/tests/test_health_snapshot.py b/packages/sandbox-runtime/tests/test_health_snapshot.py new file mode 100644 index 0000000000..02638001ec --- /dev/null +++ b/packages/sandbox-runtime/tests/test_health_snapshot.py @@ -0,0 +1,39 @@ +from sandbox_runtime import health_snapshot + + +def test_reads_available_linux_resource_counters(tmp_path, monkeypatch): + files = { + "_MEMINFO": "MemTotal: 4096000 kB\nMemAvailable: 1536000 kB\n", + "_MEMORY_PRESSURE": "some avg10=12.50 avg60=2.00 total=1\nfull avg10=0.00\n", + "_CGROUP_MEMORY_CURRENT": "2147483648\n", + "_CGROUP_MEMORY_MAX": "4294967296\n", + "_CGROUP_MEMORY_EVENTS": "low 0\noom 3\noom_kill 1\n", + } + for name, content in files.items(): + path = tmp_path / name + path.write_text(content) + monkeypatch.setattr(health_snapshot, name, path) + + assert health_snapshot.read_health_snapshot() == { + "memory_available_mib": 1500, + "memory_psi_some_avg10": 12.5, + "cgroup_memory_current_mib": 2048, + "cgroup_memory_max_mib": 4096, + "cgroup_oom_count": 3, + "cgroup_oom_kill_count": 1, + } + + +def test_missing_or_unlimited_counters_are_omitted(tmp_path, monkeypatch): + for name in ( + "_MEMINFO", + "_MEMORY_PRESSURE", + "_CGROUP_MEMORY_CURRENT", + "_CGROUP_MEMORY_EVENTS", + ): + monkeypatch.setattr(health_snapshot, name, tmp_path / name) + maximum = tmp_path / "memory.max" + maximum.write_text("max\n") + monkeypatch.setattr(health_snapshot, "_CGROUP_MEMORY_MAX", maximum) + + assert health_snapshot.read_health_snapshot() == {} From 780e3af03085b11cee58e5eae912c415ef926f3f Mon Sep 17 00:00:00 2001 From: Cole Murray Date: Wed, 23 Sep 2026 09:48:04 -0700 Subject: [PATCH 08/22] Update lifecycle and bridge log assertions (cherry picked from commit b7783bf02e4544dfcaeb85654407cedacf55329f) --- .../src/sandbox/lifecycle/alarm-inactivity-effects.test.ts | 4 ++-- packages/sandbox-runtime/tests/test_bridge_reconnection.py | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/control-plane/src/sandbox/lifecycle/alarm-inactivity-effects.test.ts b/packages/control-plane/src/sandbox/lifecycle/alarm-inactivity-effects.test.ts index 15f5b79d19..d50fc2e5d6 100644 --- a/packages/control-plane/src/sandbox/lifecycle/alarm-inactivity-effects.test.ts +++ b/packages/control-plane/src/sandbox/lifecycle/alarm-inactivity-effects.test.ts @@ -10,10 +10,10 @@ describe("inactivity alarm effects", () => { it.each([ { - name: "remaining inactivity", + name: "heartbeat before remaining inactivity", ageMs: 120_000, clients: 0, - delayMs: DEFAULT_LIFECYCLE_CONFIG.inactivity.timeoutMs - 120_000, + delayMs: DEFAULT_LIFECYCLE_CONFIG.heartbeat.timeoutMs + 1, }, { name: "minimum interval", diff --git a/packages/sandbox-runtime/tests/test_bridge_reconnection.py b/packages/sandbox-runtime/tests/test_bridge_reconnection.py index 3734af644e..2ba3296066 100644 --- a/packages/sandbox-runtime/tests/test_bridge_reconnection.py +++ b/packages/sandbox-runtime/tests/test_bridge_reconnection.py @@ -98,6 +98,9 @@ async def __aexit__(self, *_args): "sandbox_runtime.bridge.websockets.connect", lambda *_args, **_kwargs: ConnectionContext(ws), ) + monkeypatch.setattr( + "sandbox_runtime.bridge.read_health_snapshot", lambda: {"memory_available_mib": 1500} + ) bridge.log = MagicMock() bridge._send_event = AsyncMock(side_effect=asyncio.CancelledError) @@ -114,6 +117,7 @@ async def __aexit__(self, *_args): connection_count=1, reconnect_count=0, reconnect_attempt_count=0, + memory_available_mib=1500, ) @pytest.mark.asyncio From 8126a94ff3115068c1825d8c24803b25cfab6f11 Mon Sep 17 00:00:00 2001 From: Cole Murray Date: Wed, 23 Sep 2026 10:32:07 -0700 Subject: [PATCH 09/22] fix: block prompts during failed sandbox preservation (cherry picked from commit ed2f9a70bb0441d3a5aa09aeef9bf5f866359793) --- .../control-plane/src/session/components.ts | 4 +- .../handlers/child-sessions.handler.test.ts | 23 ++++- .../http/handlers/child-sessions.handler.ts | 12 ++- .../http/handlers/messages.handler.test.ts | 21 +++++ .../session/http/handlers/messages.handler.ts | 7 ++ .../src/session/message-queue.test.ts | 91 ++++++++++++++++++- .../src/session/message-queue.ts | 30 +++++- .../src/session/message-repository.test.ts | 23 +++++ .../src/session/message-repository.ts | 11 ++- packages/shared/src/types/github-autofix.ts | 8 +- .../shared/src/types/sandbox-shutdown.test.ts | 34 ++++++- packages/shared/src/types/sandbox-shutdown.ts | 10 ++ .../app/(app)/(sidebar)/session/[id]/page.tsx | 21 +++-- .../sandbox-shutdown-banner.test.tsx | 1 + .../components/sandbox-shutdown-banner.tsx | 4 + 15 files changed, 283 insertions(+), 17 deletions(-) diff --git a/packages/control-plane/src/session/components.ts b/packages/control-plane/src/session/components.ts index 0669e25a17..c314b6673a 100644 --- a/packages/control-plane/src/session/components.ts +++ b/packages/control-plane/src/session/components.ts @@ -23,6 +23,7 @@ import { resolveAppName } from "@open-inspect/shared/app-name"; import { DEFAULT_MODEL } from "@open-inspect/shared/models"; +import { sandboxPromptBlockReason } from "@open-inspect/shared/types/sandbox-shutdown"; import { generateId, hashToken, encryptToken } from "../auth/crypto"; import { getUserAuth } from "../auth/user/runtime"; import { resolveSandboxBackendName } from "../sandbox/provider-name"; @@ -502,7 +503,8 @@ export function createSessionRuntime(platform: SessionPlatform, env: Env): Sessi alarmScheduler, executionStop, getExecutionTimeoutMs, - () => lifecycleManager.mayProcessQueuedWork() + () => lifecycleManager.mayProcessQueuedWork(), + () => sandboxPromptBlockReason(lifecycleManager.shutdownSnapshot()) ); // Tier 7 — services over the queue and lifecycle. diff --git a/packages/control-plane/src/session/http/handlers/child-sessions.handler.test.ts b/packages/control-plane/src/session/http/handlers/child-sessions.handler.test.ts index 023b6dfbfb..8e4eb32e19 100644 --- a/packages/control-plane/src/session/http/handlers/child-sessions.handler.test.ts +++ b/packages/control-plane/src/session/http/handlers/child-sessions.handler.test.ts @@ -1,7 +1,11 @@ import { describe, expect, it, vi } from "vitest"; import { MAX_CHILD_FOLLOW_UP_PROMPT_CHARS } from "@open-inspect/shared/types/session-api"; import { ChildSessionsHandler } from "./child-sessions.handler"; -import { PromptQueueFullError, SessionNotPromptableError } from "../../message-queue"; +import { + PromptQueueFullError, + SandboxPromptBlockedError, + SessionNotPromptableError, +} from "../../message-queue"; import type { ParticipantRow, SessionRow } from "../../types"; import type { ParticipantRepository } from "../../participant-repository"; import type { MessageRepository } from "../../message-repository"; @@ -234,6 +238,23 @@ describe("ChildSessionsHandler", () => { error: "Cannot prompt a archived session", }); }); + + it("maps a sandbox safety hold to 409", async () => { + const { handler, getSession, repository, enqueuePrompt } = createHandler(); + getSession.mockReturnValue(createSession({ parent_session_id: "parent-1" })); + repository.listParticipants.mockReturnValue([createParticipant()]); + enqueuePrompt.mockRejectedValue(new SandboxPromptBlockedError("Start a new session.")); + + const response = await handler.parentPrompt( + request({ parentSessionId: "parent-1", content: "Continue" }) + ); + + expect(response.status).toBe(409); + await expect(response.json()).resolves.toEqual({ + error: "Start a new session.", + code: "SANDBOX_RECOVERY_REQUIRED", + }); + }); }); it("returns 404 when session is missing for spawn context", async () => { diff --git a/packages/control-plane/src/session/http/handlers/child-sessions.handler.ts b/packages/control-plane/src/session/http/handlers/child-sessions.handler.ts index 211696d90d..d7ceae50bb 100644 --- a/packages/control-plane/src/session/http/handlers/child-sessions.handler.ts +++ b/packages/control-plane/src/session/http/handlers/child-sessions.handler.ts @@ -5,7 +5,11 @@ import { z } from "zod"; import { sessionStatusSchema } from "@open-inspect/shared/types/sessions"; import { parsePersistedSandboxSettings } from "../../../sandbox/settings"; import type { SessionMessenger } from "../../messenger"; -import { PromptQueueFullError, SessionNotPromptableError } from "../../message-queue"; +import { + PromptQueueFullError, + SandboxPromptBlockedError, + SessionNotPromptableError, +} from "../../message-queue"; import type { MessageRepository } from "../../message-repository"; import type { ParticipantRepository } from "../../participant-repository"; import type { SessionCoreRepository } from "../../session-core-repository"; @@ -165,6 +169,12 @@ export class ChildSessionsHandler { if (error instanceof SessionNotPromptableError) { return Response.json({ error: error.message }, { status: 409 }); } + if (error instanceof SandboxPromptBlockedError) { + return Response.json( + { error: error.message, code: "SANDBOX_RECOVERY_REQUIRED" }, + { status: 409 } + ); + } if (error instanceof PromptQueueFullError) { return Response.json({ error: "Child prompt queue is full" }, { status: 429 }); } diff --git a/packages/control-plane/src/session/http/handlers/messages.handler.test.ts b/packages/control-plane/src/session/http/handlers/messages.handler.test.ts index 76c78ed525..7e8528e7a0 100644 --- a/packages/control-plane/src/session/http/handlers/messages.handler.test.ts +++ b/packages/control-plane/src/session/http/handlers/messages.handler.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it, vi } from "vitest"; import type { Logger } from "../../../logger"; import { MessagesHandler } from "./messages.handler"; +import { SandboxPromptBlockedError } from "../../message-queue"; import type { MessageService } from "../../services/message.service"; function createHandler() { @@ -29,6 +30,26 @@ function createHandler() { } describe("MessagesHandler", () => { + it("returns a recoverable 409 when sandbox safety blocks prompt admission", async () => { + const { handler, messageService, log } = createHandler(); + vi.mocked(messageService.enqueuePrompt).mockRejectedValue( + new SandboxPromptBlockedError("Start a new session to continue.") + ); + + const response = await handler.enqueuePrompt( + new Request("http://internal/internal/prompt", { + method: "POST", + body: JSON.stringify({ content: "Continue", authorId: "user-1", source: "web" }), + }), + log + ); + + expect(response.status).toBe(409); + await expect(response.json()).resolves.toEqual({ + code: "SANDBOX_RECOVERY_REQUIRED", + error: "Start a new session to continue.", + }); + }); it("enqueues prompt and returns queued response", async () => { const { handler, messageService, log } = createHandler(); vi.mocked(messageService.enqueuePrompt).mockResolvedValue({ diff --git a/packages/control-plane/src/session/http/handlers/messages.handler.ts b/packages/control-plane/src/session/http/handlers/messages.handler.ts index ce72189b3b..e631a99367 100644 --- a/packages/control-plane/src/session/http/handlers/messages.handler.ts +++ b/packages/control-plane/src/session/http/handlers/messages.handler.ts @@ -15,6 +15,7 @@ import { HarnessModelIncompatibleError, PromptRequestConflictError, SessionNotPromptableError, + SandboxPromptBlockedError, } from "../../message-queue"; /** @@ -42,6 +43,12 @@ export class MessagesHandler { if (error instanceof SessionNotPromptableError) { return Response.json({ error: error.message }, { status: 409 }); } + if (error instanceof SandboxPromptBlockedError) { + return Response.json( + { error: error.message, code: "SANDBOX_RECOVERY_REQUIRED" }, + { status: 409 } + ); + } if (error instanceof BudgetExhaustedError) { return Response.json({ error: error.message, code: "BUDGET_EXHAUSTED" }, { status: 409 }); } diff --git a/packages/control-plane/src/session/message-queue.test.ts b/packages/control-plane/src/session/message-queue.test.ts index b06629fa3c..732cac10fd 100644 --- a/packages/control-plane/src/session/message-queue.test.ts +++ b/packages/control-plane/src/session/message-queue.test.ts @@ -136,7 +136,10 @@ it("creates a canonical SHA-256 web prompt fingerprint", async () => { ).resolves.toBe(fingerprint); }); -function buildQueue(mayDispatch: () => boolean = () => true) { +function buildQueue( + mayDispatch: () => boolean = () => true, + getSandboxPromptBlockReason: () => string | null = () => null +) { // Mutable so tests can pin that the deadline honors the value current at // dispatch time — the thunk exists because settings can be persisted after // the queue is constructed. @@ -299,7 +302,8 @@ function buildQueue(mayDispatch: () => boolean = () => true) { alarmScheduler, executionStop, () => executionTimeoutMs, - mayDispatch + mayDispatch, + getSandboxPromptBlockReason ); return { @@ -328,6 +332,56 @@ function buildQueue(mayDispatch: () => boolean = () => true) { } describe("SessionMessageQueue", () => { + it("rejects new websocket and API prompts during a failed safety hold", async () => { + const h = buildQueue( + () => false, + () => "Start a new session to continue." + ); + const ws = {} as WebSocket; + h.participantService.getByUserId.mockReturnValue(null as unknown as ParticipantRow); + + await h.queue.handlePromptMessage(ws, createClientInfo(), { + content: "Continue", + clientRequestId: "request-1", + }); + await expect( + h.queue.enqueuePromptFromApi({ content: "Continue", authorId: "user-1", source: "agent" }) + ).rejects.toMatchObject({ name: "SandboxPromptBlockedError" }); + + expect(h.wsManager.send).toHaveBeenCalledWith( + ws, + expect.objectContaining({ + type: "error", + code: "SANDBOX_RECOVERY_REQUIRED", + clientRequestId: "request-1", + message: "Start a new session to continue.", + }) + ); + expect(h.participantService.create).not.toHaveBeenCalled(); + expect(h.repository.createMessageWithAttachments).not.toHaveBeenCalled(); + expect(h.sessionStatus.transition).not.toHaveBeenCalled(); + }); + + it("rechecks the safety hold after asynchronous prompt fingerprinting", async () => { + let held = false; + const h = buildQueue( + () => true, + () => (held ? "Sandbox recovery required" : null) + ); + const ws = {} as WebSocket; + const handling = h.queue.handlePromptMessage(ws, createClientInfo(), { + content: "Continue", + clientRequestId: "request-1", + }); + held = true; + await handling; + + expect(h.repository.createMessageWithAttachments).not.toHaveBeenCalled(); + expect(h.wsManager.send).toHaveBeenCalledWith( + ws, + expect.objectContaining({ code: "SANDBOX_RECOVERY_REQUIRED" }) + ); + }); it("cannot dispatch while final-cost settlement waits for terminal projection", async () => { const h = buildQueue(); const session = createSession({ total_cost: 9, max_cost_usd: 10 }); @@ -445,6 +499,7 @@ describe("SessionMessageQueue", () => { attemptLimit: 10, windowStart: expect.any(Number), sessionClosed: false, + sandboxRecoveryRequired: false, }); expect(h.repository.createEvent).not.toHaveBeenCalled(); expect(h.sessionStatus.transition).toHaveBeenCalledWith("active"); @@ -510,6 +565,38 @@ describe("SessionMessageQueue", () => { expect(h.repository.updateParticipantCoalesce).not.toHaveBeenCalled(); }); + it("rejects new Autofix feedback during a failed safety hold", async () => { + const h = buildQueue( + () => false, + () => "Sandbox recovery required" + ); + h.repository.admitAutofixMessage.mockReturnValue({ + kind: "rejected", + reason: "sandbox_recovery_required", + }); + + const result = await h.queue.enqueueAutofix({ + type: "enqueue_feedback", + feedbackKey: "github:review:held", + pullRequest: { repositoryId: "99", number: 42, artifactId: "artifact-1" }, + prompt: "Address the feedback", + author: { id: "7", login: "alice" }, + origin: { + kind: "review", + authorType: "human", + feedbackUrl: "https://github.com/acme/widgets/pull/42#pullrequestreview-held", + }, + attemptLimit: 10, + }); + + expect(result).toEqual({ kind: "rejected", reason: "sandbox_recovery_required" }); + expect(h.repository.admitAutofixMessage).toHaveBeenCalledWith( + expect.objectContaining({ sandboxRecoveryRequired: true }) + ); + expect(h.participantService.create).not.toHaveBeenCalled(); + expect(h.sessionStatus.transition).not.toHaveBeenCalled(); + }); + it("returns a duplicate without re-driving it in a closed session", async () => { const h = buildQueue(); h.repository.getSession.mockReturnValue(createSession({ status: "archived" })); diff --git a/packages/control-plane/src/session/message-queue.ts b/packages/control-plane/src/session/message-queue.ts index fdb88c01f2..54ca189d09 100644 --- a/packages/control-plane/src/session/message-queue.ts +++ b/packages/control-plane/src/session/message-queue.ts @@ -77,6 +77,13 @@ export class SessionNotPromptableError extends Error { } } +export class SandboxPromptBlockedError extends Error { + constructor(message: string) { + super(message); + this.name = "SandboxPromptBlockedError"; + } +} + export class BudgetExhaustedError extends Error { constructor() { super( @@ -164,13 +171,15 @@ export class SessionMessageQueue { private readonly executionStop: ExecutionStopCoordinator, /** Resolved per use so it honors settings persisted after construction. */ private readonly getExecutionTimeoutMs: () => number, - private readonly mayDispatch: () => boolean = () => true + private readonly mayDispatch: () => boolean, + private readonly getSandboxPromptBlockReason: () => string | null ) {} async enqueueAutofix( command: Extract ): Promise { const session = this.repository.getSession(); + const sandboxRecoveryRequired = this.getSandboxPromptBlockReason() !== null; const userId = `github:${command.author.id}`; const now = Date.now(); const admission = this.messageRepository.admitAutofixMessage({ @@ -199,6 +208,7 @@ export class SessionMessageQueue { attemptLimit: command.attemptLimit, windowStart: now - AUTOFIX_ATTEMPT_WINDOW_MS, sessionClosed: !session || session.status === "archived" || session.status === "cancelled", + sandboxRecoveryRequired, }); if (admission.kind === "rejected") return admission; @@ -229,6 +239,7 @@ export class SessionMessageQueue { const session = this.repository.getSession(); if (!session || session.status === "archived" || session.status === "cancelled") return; + if (this.getSandboxPromptBlockReason()) return; await this.sessionStatus.transition("active"); await this.processMessageQueue(); @@ -242,6 +253,7 @@ export class SessionMessageQueue { let enqueued: EnqueuedPrompt; try { this.assertPromptableSession(); + this.assertSandboxAcceptingPrompts(); let participant = this.participantRepository.getParticipantById(client.participantId); participant ??= this.participantService.getByUserId(client.userId); if (!participant) { @@ -278,6 +290,15 @@ export class SessionMessageQueue { }); return; } + if (error instanceof SandboxPromptBlockedError) { + this.wsManager.send(ws, { + type: "error", + code: "SANDBOX_RECOVERY_REQUIRED", + message: error.message, + clientRequestId: data.clientRequestId, + }); + return; + } if (error instanceof PromptQueueFullError) { this.wsManager.send(ws, { type: "error", @@ -679,6 +700,7 @@ export class SessionMessageQueue { data: EnqueuePromptRequest ): Promise<{ messageId: string; status: "queued" }> { this.assertPromptableSession(); + this.assertSandboxAcceptingPrompts(); this.assertBudgetAvailable(); this.assertQueueCapacity(); let participant = this.participantService.getByUserId(data.authorId); @@ -738,6 +760,7 @@ export class SessionMessageQueue { // cancel or archive can land while this request is suspended, so the // session is read after it, not before. this.assertPromptableSession(); + this.assertSandboxAcceptingPrompts(); const queueDepthBefore = this.messageRepository.getPendingOrProcessingCount(); if (data.clientRequestId) { const existing = this.messageRepository.getMessageByClientRequestId(data.clientRequestId); @@ -878,6 +901,11 @@ export class SessionMessageQueue { } } + private assertSandboxAcceptingPrompts(): void { + const reason = this.getSandboxPromptBlockReason(); + if (reason) throw new SandboxPromptBlockedError(reason); + } + private assertQueueCapacity( queueDepth = this.messageRepository.getPendingOrProcessingCount() ): void { diff --git a/packages/control-plane/src/session/message-repository.test.ts b/packages/control-plane/src/session/message-repository.test.ts index 545739f2e4..5ffd590532 100644 --- a/packages/control-plane/src/session/message-repository.test.ts +++ b/packages/control-plane/src/session/message-repository.test.ts @@ -286,6 +286,29 @@ describe("MessageRepository", () => { expect(mock.calls).toHaveLength(2); }); + it("rejects new Autofix feedback during a sandbox safety hold", () => { + expect( + repository.admitAutofixMessage({ + message: { + id: "msg-new", + authorId: "p-1", + content: "Fix feedback", + source: "github", + status: "pending", + createdAt: 2000, + }, + feedbackKey: "github:review:held", + pullRequestKey: "github:99:42", + originContext: "{}", + attemptLimit: 3, + windowStart: 1000, + sessionClosed: false, + sandboxRecoveryRequired: true, + }) + ).toEqual({ kind: "rejected", reason: "sandbox_recovery_required" }); + expect(mock.calls).toHaveLength(2); + }); + it("rejects new Autofix feedback when the session budget is exhausted", () => { mock.setData(`SELECT budget_exhausted FROM session LIMIT 1`, [{ budget_exhausted: 1 }]); diff --git a/packages/control-plane/src/session/message-repository.ts b/packages/control-plane/src/session/message-repository.ts index 99a77d12b5..47fa38975e 100644 --- a/packages/control-plane/src/session/message-repository.ts +++ b/packages/control-plane/src/session/message-repository.ts @@ -68,6 +68,7 @@ export interface AdmitAutofixMessageData { attemptLimit: number | null; windowStart: number; sessionClosed: boolean; + sandboxRecoveryRequired?: boolean; } export type AutofixMessageAdmission = @@ -75,7 +76,12 @@ export type AutofixMessageAdmission = | { kind: "duplicate"; messageId: string } | { kind: "rejected"; - reason: "session_closed" | "budget_exhausted" | "queue_full" | "attempt_limit"; + reason: + | "session_closed" + | "sandbox_recovery_required" + | "budget_exhausted" + | "queue_full" + | "attempt_limit"; }; /** Options for listing messages. */ @@ -237,6 +243,9 @@ export class MessageRepository { if (data.sessionClosed) { return { kind: "rejected", reason: "session_closed" }; } + if (data.sandboxRecoveryRequired) { + return { kind: "rejected", reason: "sandbox_recovery_required" }; + } if (this.getPendingOrProcessingCount() >= MAX_UNFINISHED_PROMPTS) { return { kind: "rejected", reason: "queue_full" }; } diff --git a/packages/shared/src/types/github-autofix.ts b/packages/shared/src/types/github-autofix.ts index ab2a85f2e8..5bfe77b0e8 100644 --- a/packages/shared/src/types/github-autofix.ts +++ b/packages/shared/src/types/github-autofix.ts @@ -145,7 +145,13 @@ export const githubAutofixSessionResponseSchema = z.discriminatedUnion("kind", [ }), z.object({ kind: z.literal("rejected"), - reason: z.enum(["session_closed", "budget_exhausted", "queue_full", "attempt_limit"]), + reason: z.enum([ + "session_closed", + "sandbox_recovery_required", + "budget_exhausted", + "queue_full", + "attempt_limit", + ]), }), z.object({ kind: z.literal("found"), diff --git a/packages/shared/src/types/sandbox-shutdown.test.ts b/packages/shared/src/types/sandbox-shutdown.test.ts index a6c4d42a63..fdef5bf1d0 100644 --- a/packages/shared/src/types/sandbox-shutdown.test.ts +++ b/packages/shared/src/types/sandbox-shutdown.test.ts @@ -1,5 +1,9 @@ import { describe, expect, it } from "vitest"; -import { sandboxShutdownSchema } from "./sandbox-shutdown"; +import { + sandboxPromptBlockReason, + sandboxShutdownSchema, + type SandboxShutdownState, +} from "./sandbox-shutdown"; describe("sandboxShutdownSchema", () => { it("round-trips authoritative recovery actions while remaining rolling-compatible", () => { @@ -16,3 +20,31 @@ describe("sandboxShutdownSchema", () => { ).toBe(false); }); }); + +const state = (phase: SandboxShutdownState["phase"]): SandboxShutdownState => ({ + phase, + expiresAtMs: null, + drainAtMs: null, +}); + +describe("sandboxPromptBlockReason", () => { + it.each(["failed", "unknown"] as const)( + "blocks prompts in %s without a recovery action", + (phase) => { + expect(sandboxPromptBlockReason(state(phase))).toContain("start a new session"); + } + ); + + it("points to recovery when one is available", () => { + expect( + sandboxPromptBlockReason({ ...state("failed"), availableRecoveryActions: ["retry"] }) + ).toContain("Use an available recovery action"); + }); + + it.each(["running", "draining", "capturing", "saved", "restoring"] as const)( + "allows existing prompt queue behavior in %s", + (phase) => { + expect(sandboxPromptBlockReason(state(phase))).toBeNull(); + } + ); +}); diff --git a/packages/shared/src/types/sandbox-shutdown.ts b/packages/shared/src/types/sandbox-shutdown.ts index 845cb11001..741e86119b 100644 --- a/packages/shared/src/types/sandbox-shutdown.ts +++ b/packages/shared/src/types/sandbox-shutdown.ts @@ -29,3 +29,13 @@ export const sandboxShutdownSchema = z.object({ }); export type SandboxShutdownState = z.infer; + +/** Failed/unknown shutdowns hold work until an explicit recovery succeeds. */ +export function sandboxPromptBlockReason( + state: SandboxShutdownState | null | undefined +): string | null { + if (state?.phase !== "failed" && state?.phase !== "unknown") return null; + return state.availableRecoveryActions?.length + ? "New prompts are blocked until the sandbox is recovered. Use an available recovery action to continue." + : "New prompts are blocked. No recovery action is currently available; start a new session to continue. Unsaved changes may be missing."; +} diff --git a/packages/web/src/app/(app)/(sidebar)/session/[id]/page.tsx b/packages/web/src/app/(app)/(sidebar)/session/[id]/page.tsx index ca107807a1..a6e9574582 100644 --- a/packages/web/src/app/(app)/(sidebar)/session/[id]/page.tsx +++ b/packages/web/src/app/(app)/(sidebar)/session/[id]/page.tsx @@ -66,6 +66,7 @@ import { useSessionRename } from "@/hooks/use-session-rename"; import { useCurrentUserAuthorization } from "@/hooks/use-current-user-authorization"; import { resolveSessionCapabilities } from "@/lib/session-capabilities"; import { SandboxShutdownBanner } from "@/components/sandbox-shutdown-banner"; +import { sandboxPromptBlockReason } from "@open-inspect/shared/types/sandbox-shutdown"; type SessionState = ReturnType["sessionState"]; @@ -128,6 +129,7 @@ export default function SessionPage() { }); // Fixed at create; per-message model overrides must stay within it. const sessionHarness = sessionState?.harness ?? initialSnapshot.session.harness; + const sandboxBlockReason = sandboxPromptBlockReason(sessionState?.sandboxPreservation); const { selectedModel, reasoningEffort, @@ -156,7 +158,7 @@ export default function SessionPage() { reasoningEffort, loadingEnabledModels, sessionState?.status ?? DEFAULT_SESSION_STATUS, - ready && capabilities.collaborate && !sessionState?.budgetExhausted, + ready && capabilities.collaborate && !sessionState?.budgetExhausted && !sandboxBlockReason, shortcuts["send-prompt"] ); const [cancellingPromptIds, setCancellingPromptIds] = useState>(new Set()); @@ -378,15 +380,18 @@ export default function SessionPage() { draftLocked: isSubmitting || sessionAttachments.isUploading, sendBlocked: !ready || + Boolean(sandboxBlockReason) || Boolean(sessionState?.budgetExhausted) || modelAvailability.status === "unavailable", - blockedReason: sessionState?.budgetExhausted - ? canManageBudget - ? `Session cost limit reached at ${formatSessionCost(sessionState.totalCost ?? 0)} of ${formatSessionCost(sessionState.maxSessionCostUsd ?? 0)}. Raise or remove the limit to continue.` - : `Session cost limit reached at ${formatSessionCost(sessionState.totalCost ?? 0)} of ${formatSessionCost(sessionState.maxSessionCostUsd ?? 0)}. The session owner must raise or remove the limit to continue.` - : modelAvailability.status === "unavailable" - ? modelAvailability.message - : undefined, + blockedReason: + sandboxBlockReason ?? + (sessionState?.budgetExhausted + ? canManageBudget + ? `Session cost limit reached at ${formatSessionCost(sessionState.totalCost ?? 0)} of ${formatSessionCost(sessionState.maxSessionCostUsd ?? 0)}. Raise or remove the limit to continue.` + : `Session cost limit reached at ${formatSessionCost(sessionState.totalCost ?? 0)} of ${formatSessionCost(sessionState.maxSessionCostUsd ?? 0)}. The session owner must raise or remove the limit to continue.` + : modelAvailability.status === "unavailable" + ? modelAvailability.message + : undefined), submitError, inputRef, onSubmit: handleSubmit, diff --git a/packages/web/src/components/sandbox-shutdown-banner.test.tsx b/packages/web/src/components/sandbox-shutdown-banner.test.tsx index f2acce96cc..c0fb043e61 100644 --- a/packages/web/src/components/sandbox-shutdown-banner.test.tsx +++ b/packages/web/src/components/sandbox-shutdown-banner.test.tsx @@ -232,6 +232,7 @@ describe("SandboxShutdownBanner", () => { expect(screen.queryByRole("button", { name: "Retry shutdown" })).not.toBeInTheDocument(); expect(screen.queryByRole("button", { name: "Restore saved state" })).not.toBeInTheDocument(); + expect(screen.getByRole("alert")).toHaveTextContent("start a new session"); } ); diff --git a/packages/web/src/components/sandbox-shutdown-banner.tsx b/packages/web/src/components/sandbox-shutdown-banner.tsx index 546258005c..a2f4625cbb 100644 --- a/packages/web/src/components/sandbox-shutdown-banner.tsx +++ b/packages/web/src/components/sandbox-shutdown-banner.tsx @@ -6,6 +6,7 @@ import type { ShutdownRecoveryAction, } from "@open-inspect/shared/types/sandbox-shutdown"; import { cn } from "@/lib/utils"; +import { sandboxPromptBlockReason } from "@open-inspect/shared/types/sandbox-shutdown"; import type { ShutdownRecoveryResult } from "@/hooks/use-session-socket"; const PHASE_MESSAGES: Record, string> = { @@ -81,6 +82,9 @@ export function SandboxShutdownBanner({ shutdown, onRecover }: SandboxShutdownBa )} {detail && {detail}} + {isError && recoveryActions.length === 0 && ( + {sandboxPromptBlockReason(shutdown)} + )} {shutdown.phase === "failed" && canRetry && onRecover && ( )} {isError && canRestoreSaved && onRecover && ( @@ -113,6 +121,18 @@ export function SandboxShutdownBanner({ shutdown, onRecover }: SandboxShutdownBa {pendingAction === "restore_saved" ? "Restoring saved state…" : "Restore saved state"} )} + {isError && canDiscard && onRecover && ( + + )} {canResumeQueuedWork && onRecover && (