diff --git a/crates/alien-test/src/e2e.rs b/crates/alien-test/src/e2e.rs index 99ea3fa81..67340a8ce 100644 --- a/crates/alien-test/src/e2e.rs +++ b/crates/alien-test/src/e2e.rs @@ -182,6 +182,8 @@ pub enum Binding { Vault, /// Managed Postgres database (Aurora, Cloud SQL, Flexible Server, embedded pgvector on Local) Postgres, + /// Sandbox session: run a command, move a file in and out, terminate + Sandbox, /// Message queue (SQS, Pub/Sub, Service Bus) Queue, /// Direct worker-to-worker invocation @@ -225,6 +227,7 @@ impl std::fmt::Display for Binding { Binding::Kv => write!(f, "kv"), Binding::Vault => write!(f, "vault"), Binding::Postgres => write!(f, "postgres"), + Binding::Sandbox => write!(f, "sandbox"), Binding::Queue => write!(f, "queue"), Binding::Worker => write!(f, "worker"), Binding::Container => write!(f, "container"), @@ -304,6 +307,9 @@ pub fn supported_bindings(platform: Platform, model: DeploymentModel) -> Vec {} } diff --git a/crates/alien-test/tests/common/bindings.rs b/crates/alien-test/tests/common/bindings.rs index e5b277724..05c77bb0f 100644 --- a/crates/alien-test/tests/common/bindings.rs +++ b/crates/alien-test/tests/common/bindings.rs @@ -12,6 +12,7 @@ use tracing::info; /// The binding name used in test app stack configurations. pub(super) const STORAGE_BINDING: &str = "alien-storage"; const KV_BINDING: &str = "alien-kv"; +const SANDBOX_BINDING: &str = "alien-sandbox"; const VAULT_BINDING: &str = "alien-vault"; const POSTGRES_BINDING: &str = "alien-postgres"; const QUEUE_BINDING: &str = "alien-queue"; @@ -1081,3 +1082,43 @@ pub async fn check_ai(deployment: &TestDeployment) -> anyhow::Result<()> { ); Ok(()) } + +/// Runs a real sandbox session through the deployed app. +/// +/// The app creates a session, runs a command, moves a file both directions and terminates. A +/// failure anywhere in that chain comes back as a non-2xx, so the assertion here is the whole +/// chain rather than any one call. +pub async fn check_sandbox(deployment: &TestDeployment) -> anyhow::Result<()> { + let url = deployment_url(deployment)?; + info!("Checking sandbox binding"); + + let client = reqwest::Client::new(); + let resp = post_empty(&client, format!("{}/sandbox-test/{}", url, SANDBOX_BINDING)) + .send() + .await + .context("Sandbox test request failed")?; + + let status = resp.status(); + if !status.is_success() { + let body = resp.text().await.unwrap_or_default(); + bail!("Sandbox test returned {}: {}", status, body); + } + + let data: BindingTestResponse = resp + .json() + .await + .context("Failed to parse sandbox response")?; + if !data.success { + bail!("Sandbox test reported failure"); + } + if data.binding_name != SANDBOX_BINDING { + bail!( + "Sandbox test binding mismatch: expected {}, got {}", + SANDBOX_BINDING, + data.binding_name + ); + } + + info!("Sandbox binding check passed"); + Ok(()) +} diff --git a/crates/alien-test/tests/common/runner.rs b/crates/alien-test/tests/common/runner.rs index 7d2b75508..7c1881118 100644 --- a/crates/alien-test/tests/common/runner.rs +++ b/crates/alien-test/tests/common/runner.rs @@ -60,6 +60,7 @@ pub async fn check_all_bindings( Binding::Kv => bindings::check_kv(deployment).await?, Binding::Vault => bindings::check_vault(deployment).await?, Binding::Postgres => bindings::check_postgres(deployment).await?, + Binding::Sandbox => bindings::check_sandbox(deployment).await?, Binding::Queue => bindings::check_queue(deployment).await?, Binding::Worker => bindings::check_worker(deployment).await?, Binding::Container => bindings::check_container(deployment).await?, diff --git a/tests/e2e/test-apps/comprehensive-rust/alien.ts b/tests/e2e/test-apps/comprehensive-rust/alien.ts index 47ddf235d..f93555cb6 100644 --- a/tests/e2e/test-apps/comprehensive-rust/alien.ts +++ b/tests/e2e/test-apps/comprehensive-rust/alien.ts @@ -20,6 +20,17 @@ const queue = new alien.Queue("alien-queue").build() const eventsQueue = new alien.Queue("alien-events-queue").build() const serviceAccount = new alien.ServiceAccount("test-alien-sa").build() const postgres = isLocal ? new alien.Postgres("alien-postgres").build() : undefined +// Sandbox is Local-only here for the same reason as Postgres: the cloud sandbox controllers +// do not ship in this repo, so declaring one on a cloud target would ask the executor to +// provision a backend with no registered controller. +const sandbox = isLocal + ? new alien.Sandbox("alien-sandbox") + .code({ type: "image", image: "alpine:3.20" }) + .limits({ cpu: "500m", memory: "512Mi", disk: "1Gi", maxProcesses: 64 }) + .egress({ mode: "deny" }) + .session({ maxLifetimeSeconds: 600 }) + .build() + : undefined let workerBuilder = new alien.Worker("alien-rs-worker") .code({ @@ -52,6 +63,9 @@ let workerBuilder = new alien.Worker("alien-rs-worker") if (postgres) { workerBuilder = workerBuilder.link(postgres) } +if (sandbox) { + workerBuilder = workerBuilder.link(sandbox) +} const worker = workerBuilder.build() const executionPermissions = [ @@ -70,6 +84,9 @@ const executionPermissions = [ if (postgres) { executionPermissions.push("postgres/data-access") } +if (sandbox) { + executionPermissions.push("sandbox/execute") +} let stackBuilder = new alien.Stack("alien-rs-stack") .permissions({ @@ -90,6 +107,9 @@ let stackBuilder = new alien.Stack("alien-rs-stack") if (postgres) { stackBuilder = stackBuilder.add(postgres, "live") } +if (sandbox) { + stackBuilder = stackBuilder.add(sandbox, "frozen") +} const stack = stackBuilder.add(worker, "live").build() export default stack diff --git a/tests/e2e/test-apps/comprehensive-rust/src/bin/main.rs b/tests/e2e/test-apps/comprehensive-rust/src/bin/main.rs index be52aa8a2..66c40abc8 100644 --- a/tests/e2e/test-apps/comprehensive-rust/src/bin/main.rs +++ b/tests/e2e/test-apps/comprehensive-rust/src/bin/main.rs @@ -406,6 +406,10 @@ fn build_router(app_state: AppState) -> Router { post(handlers::artifact_registry::test_artifact_registry), ) .route("/kv-test/{binding_name}", post(handlers::kv::test_kv)) + .route( + "/sandbox-test/{binding_name}", + post(handlers::sandbox::test_sandbox), + ) .route( "/queue-test/{binding_name}", post(handlers::queue::test_queue), diff --git a/tests/e2e/test-apps/comprehensive-rust/src/error.rs b/tests/e2e/test-apps/comprehensive-rust/src/error.rs index e7dbb325c..21c522274 100644 --- a/tests/e2e/test-apps/comprehensive-rust/src/error.rs +++ b/tests/e2e/test-apps/comprehensive-rust/src/error.rs @@ -135,6 +135,19 @@ pub enum ErrorData { operation: String, }, + /// Sandbox operation failed. + #[error( + code = "SANDBOX_OPERATION_FAILED", + message = "Sandbox operation failed: {operation}", + retryable = "true", + internal = "false", + http_status_code = 500 + )] + SandboxOperationFailed { + /// Description of the sandbox operation that failed + operation: String, + }, + /// KV operation failed. #[error( code = "KV_OPERATION_FAILED", diff --git a/tests/e2e/test-apps/comprehensive-rust/src/handlers/mod.rs b/tests/e2e/test-apps/comprehensive-rust/src/handlers/mod.rs index 83857ac19..baeb2afd8 100644 --- a/tests/e2e/test-apps/comprehensive-rust/src/handlers/mod.rs +++ b/tests/e2e/test-apps/comprehensive-rust/src/handlers/mod.rs @@ -8,6 +8,7 @@ pub mod kv; pub mod postgres; pub mod queue; pub mod queue_message; +pub mod sandbox; pub mod service_account; pub mod sse; pub mod storage; diff --git a/tests/e2e/test-apps/comprehensive-rust/src/handlers/sandbox.rs b/tests/e2e/test-apps/comprehensive-rust/src/handlers/sandbox.rs new file mode 100644 index 000000000..c183b6855 --- /dev/null +++ b/tests/e2e/test-apps/comprehensive-rust/src/handlers/sandbox.rs @@ -0,0 +1,196 @@ +use std::collections::BTreeMap; +use std::time::Duration; + +use axum::{ + extract::{Path, State}, + response::Json, +}; +use chrono::Utc; +use futures_util::StreamExt; +use tracing::info; + +use crate::{ + models::{AppState, KvTestResponse}, + ErrorData, Result, +}; +use alien_error::{AlienError, Context}; +use alien_sdk::traits::{ + CommandOutput, CreateSessionRequest, RunCommandRequest, Sandbox, SandboxSessionState, +}; + +/// Test a sandbox binding by running a command and moving a file through a session. +#[utoipa::path( + post, + path = "/sandbox-test/{binding_name}", + tag = "sandbox", + params( + ("binding_name" = String, Path, description = "Name of the sandbox binding to test") + ), + responses( + (status = 200, description = "Sandbox test completed", body = KvTestResponse), + (status = 400, description = "Binding not found", body = AlienError), + (status = 500, description = "Sandbox operation failed", body = AlienError), + ), + operation_id = "test_sandbox", + summary = "Test sandbox session operations", + description = "Creates a session, runs a command, writes and reads a file, then terminates" +)] +pub async fn test_sandbox( + State(app_state): State, + Path(binding_name): Path, +) -> Result> { + info!(%binding_name, "Received sandbox test request"); + + let sandbox = app_state + .ctx + .bindings() + .sandbox(&binding_name) + .await + .context(ErrorData::BindingNotFound { + binding_name: binding_name.clone(), + })?; + + let session = sandbox + .create(CreateSessionRequest { + session_id: Some(format!("e2e-{}", Utc::now().timestamp_millis())), + tenant_key: None, + env: BTreeMap::new(), + }) + .await + .context(ErrorData::SandboxOperationFailed { + operation: "create".to_string(), + })?; + + // Everything after create runs in a helper so a failure still reaches terminate below. A + // session left running is a session still billing. + let outcome = exercise(sandbox.as_ref(), &session.session_id).await; + let terminated = terminate_and_confirm(sandbox.as_ref(), &session.session_id).await; + + // A leaked session is reported first even when the exercise also failed: an exercise failure + // is a broken test, a surviving session is a billable sandbox nobody will look for. + terminated?; + outcome?; + + info!(%binding_name, "Sandbox test completed successfully"); + + Ok(Json(KvTestResponse { + binding_name, + success: true, + })) +} + +/// How long to wait for a terminate to converge before calling the session leaked. +const TERMINATE_POLL_ATTEMPTS: u32 = 15; +const TERMINATE_POLL_INTERVAL: Duration = Duration::from_secs(2); + +/// Terminates the session and reads it back to confirm it is gone. +/// +/// A successful terminate is not the same claim: the backends return once deletion is accepted, +/// so a test that stops at the return value passes while the session keeps running. +async fn terminate_and_confirm(sandbox: &dyn Sandbox, session_id: &str) -> Result<()> { + sandbox + .terminate(session_id) + .await + .context(ErrorData::SandboxOperationFailed { + operation: "terminate".to_string(), + })?; + + // Polled rather than read once: every backend returns from terminate as soon as the deletion + // is accepted, so a single read races normal convergence and would fail a teardown that was + // simply still finishing. + // A failed read inside the window is retried like a non-terminal state: the session may well + // be gone, and giving up on the first blip would fail a teardown that had already converged. + // A read that never succeeds still fails, carrying the last error rather than a bare timeout. + let mut last = String::from("unread"); + for attempt in 0..TERMINATE_POLL_ATTEMPTS { + match sandbox.get(session_id).await { + Ok(None) => return Ok(()), + Ok(Some(session)) if session.state == SandboxSessionState::Terminated => return Ok(()), + Ok(Some(session)) => last = format!("{:?}", session.state), + Err(error) => last = format!("unreadable ({error})"), + } + + if attempt + 1 < TERMINATE_POLL_ATTEMPTS { + tokio::time::sleep(TERMINATE_POLL_INTERVAL).await; + } + } + + Err(AlienError::new(ErrorData::TestValidationFailed { + reason: format!( + "session '{session_id}' is still {last} {}s after terminate; it may still be billing", + TERMINATE_POLL_ATTEMPTS * TERMINATE_POLL_INTERVAL.as_secs() as u32 + ), + })) +} + +/// The part of the test that can fail without leaking a session. +async fn exercise(sandbox: &dyn Sandbox, session_id: &str) -> Result<()> { + let marker = format!("alien-sandbox-e2e-{}", Utc::now().timestamp_millis()); + + let mut frames = sandbox + .run_command( + session_id, + RunCommandRequest { + command: vec!["/bin/echo".to_string(), marker.clone()], + working_directory: None, + env: BTreeMap::new(), + deadline: Duration::from_secs(30), + }, + ) + .await + .context(ErrorData::SandboxOperationFailed { + operation: "run_command".to_string(), + })?; + + let mut stdout = Vec::new(); + let mut exit_code = None; + while let Some(frame) = frames.next().await { + match frame.context(ErrorData::SandboxOperationFailed { + operation: "run_command stream".to_string(), + })? { + CommandOutput::Stdout { data, .. } => stdout.extend_from_slice(&data), + CommandOutput::Exit { code, .. } => exit_code = Some(code), + CommandOutput::Stderr { .. } => {} + } + } + + if exit_code != Some(0) { + return Err(AlienError::new(ErrorData::TestValidationFailed { + reason: format!("run_command exited with {exit_code:?}, expected 0"), + })); + } + + let printed = String::from_utf8_lossy(&stdout); + if !printed.contains(&marker) { + return Err(AlienError::new(ErrorData::TestValidationFailed { + reason: format!("stdout did not carry the marker, got '{printed}'"), + })); + } + + // Files both directions through the same session, which is what makes it a session rather + // than a sequence of unrelated commands. + sandbox + .write_files( + session_id, + BTreeMap::from([("e2e/input.txt".to_string(), marker.as_bytes().to_vec())]), + ) + .await + .context(ErrorData::SandboxOperationFailed { + operation: "write_files".to_string(), + })?; + + let read_back = sandbox + .read_file(session_id, "e2e/input.txt") + .await + .context(ErrorData::SandboxOperationFailed { + operation: "read_file".to_string(), + })?; + + if read_back != marker.as_bytes() { + return Err(AlienError::new(ErrorData::TestValidationFailed { + reason: "read_file returned different bytes than write_files sent".to_string(), + })); + } + + Ok(()) +} diff --git a/tests/e2e/test-apps/comprehensive-typescript/alien.ts b/tests/e2e/test-apps/comprehensive-typescript/alien.ts index fd16c9759..1aa56b930 100644 --- a/tests/e2e/test-apps/comprehensive-typescript/alien.ts +++ b/tests/e2e/test-apps/comprehensive-typescript/alien.ts @@ -18,6 +18,17 @@ const queue = new alien.Queue("alien-queue").build() // race that consumer. This queue has exactly one consumer: the queue trigger. const eventsQueue = new alien.Queue("alien-events-queue").build() const postgres = isLocal ? new alien.Postgres("alien-postgres").build() : undefined +// Sandbox is Local-only for the same reason as Postgres: the cloud sandbox controllers do not +// ship in this repo, so declaring one on a cloud target would ask the executor to provision a +// backend with no registered controller. +const sandbox = isLocal + ? new alien.Sandbox("alien-sandbox") + .code({ type: "image", image: "alpine:3.20" }) + .limits({ cpu: "500m", memory: "512Mi", disk: "1Gi", maxProcesses: 64 }) + .egress({ mode: "deny" }) + .session({ maxLifetimeSeconds: 600 }) + .build() + : undefined const ai = new alien.AI("test-ai").build() let workerBuilder = new alien.Worker("alien-ts-worker") @@ -50,6 +61,9 @@ let workerBuilder = new alien.Worker("alien-ts-worker") if (postgres) { workerBuilder = workerBuilder.link(postgres) } +if (sandbox) { + workerBuilder = workerBuilder.link(sandbox) +} const worker = workerBuilder.build() const executionPermissions = [ @@ -67,6 +81,9 @@ const executionPermissions = [ if (postgres) { executionPermissions.push("postgres/data-access") } +if (sandbox) { + executionPermissions.push("sandbox/execute") +} let stackBuilder = new alien.Stack("alien-ts-stack") .permissions({ @@ -85,6 +102,9 @@ let stackBuilder = new alien.Stack("alien-ts-stack") if (postgres) { stackBuilder = stackBuilder.add(postgres, "live") } +if (sandbox) { + stackBuilder = stackBuilder.add(sandbox, "frozen") +} const stack = stackBuilder.add(worker, "live").build() export default stack diff --git a/tests/e2e/test-apps/comprehensive-typescript/src/handlers/sandbox.ts b/tests/e2e/test-apps/comprehensive-typescript/src/handlers/sandbox.ts new file mode 100644 index 000000000..9c0a3fc25 --- /dev/null +++ b/tests/e2e/test-apps/comprehensive-typescript/src/handlers/sandbox.ts @@ -0,0 +1,114 @@ +import type { Sandbox } from "@alienplatform/sdk" +import { sandbox } from "@alienplatform/sdk" +import { Hono } from "hono" +import { toExternalOperationError } from "../helpers.js" + +const app = new Hono() + +app.post("/sandbox-test/:bindingName", async c => { + const bindingName = c.req.param("bindingName") + const box = sandbox(bindingName) + const marker = `alien-sandbox-e2e-${Date.now()}` + + const session = await box.create({ sessionId: `e2e-ts-${Date.now()}` }).catch(async error => { + throw await toExternalOperationError(error, "sandbox-test") + }) + + // Both run before either is reported, and a leaked session is reported first even when the + // exercise also failed: an exercise failure is a broken test, a surviving session is a billable + // sandbox nobody will look for. + const outcome = await attempt(() => exercise(box, session.sessionId, marker)) + const cleanup = await attempt(() => terminateAndConfirm(box, session.sessionId)) + + const failure = cleanup ?? outcome + if (failure) { + return c.json({ success: false, error: failure }, 500) + } + + return c.json({ success: true, bindingName }) +}) + +/** Runs `step`, returning why it failed rather than throwing, so both steps always run. */ +async function attempt(step: () => Promise): Promise { + try { + return await step() + } catch (error: unknown) { + const alienError = await toExternalOperationError(error, "sandbox-test") + return `${alienError.code}: ${alienError.message}` + } +} + +/** Runs a command and moves a file both directions through one session. */ +async function exercise(box: Sandbox, sessionId: string, marker: string): Promise { + let stdout = "" + let stderr = "" + let exitCode: number | undefined + + for await (const frame of box.runCommand(sessionId, ["/bin/echo", marker], { + deadlineMs: 30_000, + })) { + if (frame.kind === "stdout") stdout += frame.data.toString("utf8") + if (frame.kind === "stderr") stderr += frame.data.toString("utf8") + if (frame.kind === "exit") exitCode = frame.exitCode + } + + if (exitCode !== 0) { + // stderr is kept, not reduced to a boolean: when this fails it is the only thing that says + // why, and this harness diagnoses a deployment from the outside. + return `command exited with ${exitCode}: ${stderr}` + } + if (!stdout.includes(marker)) { + return `stdout did not carry the marker: ${stdout}` + } + + // Files both directions through the same session, which is what makes it a session rather + // than a sequence of unrelated commands. + await box.writeFiles(sessionId, { "e2e/input.txt": marker }) + const readBack = await box.readFile(sessionId, "e2e/input.txt") + if (readBack.toString("utf8") !== marker) { + return "readFile returned different bytes than writeFiles sent" + } + + return null +} + +/** How long to wait for a terminate to converge before calling the session leaked. */ +const TERMINATE_POLL_ATTEMPTS = 15 +const TERMINATE_POLL_INTERVAL_MS = 2000 + +/** + * Terminates the session and reads it back to confirm it is gone. + * + * A successful terminate is not the same claim: the backends return once deletion is accepted, + * so a test that stops at the return value passes while the session keeps running. + */ +async function terminateAndConfirm(box: Sandbox, sessionId: string): Promise { + await box.terminate(sessionId) + + // Polled rather than read once: every backend returns from terminate as soon as the deletion is + // accepted, so a single read races normal convergence and would fail a teardown that was simply + // still finishing. + // A failed read inside the window is retried like a non-terminal state: the session may well be + // gone, and giving up on the first blip would fail a teardown that had already converged. A read + // that never succeeds still fails, carrying the last error rather than a bare timeout. + let last = "unread" + for (let attempt = 0; attempt < TERMINATE_POLL_ATTEMPTS; attempt++) { + try { + const remaining = await box.get(sessionId) + if (remaining === null || remaining.state === "terminated") { + return null + } + last = remaining.state + } catch (error: unknown) { + last = `unreadable (${error instanceof Error ? error.message : String(error)})` + } + if (attempt + 1 < TERMINATE_POLL_ATTEMPTS) { + await new Promise(resolve => setTimeout(resolve, TERMINATE_POLL_INTERVAL_MS)) + } + } + + const waited = (TERMINATE_POLL_ATTEMPTS * TERMINATE_POLL_INTERVAL_MS) / 1000 + return `session '${sessionId}' is still ${last} ${waited}s after terminate; it may still be billing` +} + +export default app diff --git a/tests/e2e/test-apps/comprehensive-typescript/src/index.ts b/tests/e2e/test-apps/comprehensive-typescript/src/index.ts index a5578f195..72b22e939 100644 --- a/tests/e2e/test-apps/comprehensive-typescript/src/index.ts +++ b/tests/e2e/test-apps/comprehensive-typescript/src/index.ts @@ -16,6 +16,7 @@ import inspectRoutes from "./handlers/inspect.js" import kvRoutes from "./handlers/kv.js" import postgresRoutes from "./handlers/postgres.js" import queueRoutes from "./handlers/queue.js" +import sandboxRoutes from "./handlers/sandbox.js" import sseRoutes from "./handlers/sse.js" import storageRoutes from "./handlers/storage.js" import vaultRoutes from "./handlers/vault.js" @@ -32,6 +33,7 @@ app.route("/", inspectRoutes) app.route("/", sseRoutes) app.route("/", storageRoutes) app.route("/", kvRoutes) +app.route("/", sandboxRoutes) app.route("/", vaultRoutes) app.route("/", postgresRoutes) app.route("/", queueRoutes)