From f34c5ec333a25f2ae26f8709ed6461061b9b5ff3 Mon Sep 17 00:00:00 2001 From: Itamar Zand <133867530+ItamarZand88@users.noreply.github.com> Date: Wed, 12 Aug 2026 15:39:14 +0300 Subject: [PATCH] feat(sandbox): add the Local and Kubernetes backends and the cloud bindings --- Cargo.lock | 3 + .../src/aws/lambda_microvms.rs | 1019 +++++++++++++++++ crates/alien-aws-clients/src/aws/mod.rs | 1 + .../tests/aws_lambda_microvms_client_tests.rs | 248 ++++ crates/alien-azure-clients/src/azure/mod.rs | 2 + .../src/azure/sandbox_data_plane.rs | 333 ++++++ .../src/azure/sandbox_groups.rs | 250 ++++ crates/alien-bindings/Cargo.toml | 10 +- crates/alien-bindings/src/bindings.rs | 14 +- crates/alien-bindings/src/provider.rs | 212 ++++ crates/alien-bindings/src/providers/mod.rs | 1 + .../src/providers/sandbox/agent_protocol.rs | 498 ++++++++ .../src/providers/sandbox/aws.rs | 852 ++++++++++++++ .../src/providers/sandbox/azure.rs | 467 ++++++++ .../src/providers/sandbox/gcp.rs | 554 +++++++++ .../src/providers/sandbox/kubernetes.rs | 349 ++++++ .../src/providers/sandbox/local.rs | 428 +++++++ .../src/providers/sandbox/mod.rs | 24 + crates/alien-bindings/src/traits.rs | 3 +- crates/alien-core/src/bin/schema_exporter.rs | 2 + crates/alien-core/src/import/data/aws/mod.rs | 2 + .../alien-core/src/import/data/aws/sandbox.rs | 30 + .../alien-core/src/import/data/azure/mod.rs | 2 + .../src/import/data/azure/sandbox.rs | 19 + crates/alien-core/src/import/data/mod.rs | 4 + ...schema_snapshots__import_data_schemas.snap | 101 ++ crates/alien-gcp-clients/src/gcp/cloudrun.rs | 41 + crates/alien-infra/Cargo.toml | 4 +- crates/alien-infra/src/core/controller.rs | 6 + .../alien-infra/src/core/controller_test.rs | 28 +- crates/alien-infra/src/core/registry.rs | 16 + .../alien-infra/src/core/service_provider.rs | 124 +- crates/alien-infra/src/kubeconfig.rs | 154 +++ crates/alien-infra/src/lib.rs | 4 + crates/alien-infra/src/network/aws_import.rs | 161 +++ crates/alien-infra/src/sandbox/kubernetes.rs | 603 ++++++++++ .../src/sandbox/kubernetes_broker.rs | 561 +++++++++ .../src/sandbox/kubernetes_eligibility.rs | 200 ++++ .../src/sandbox/kubernetes_route.rs | 303 +++++ .../src/sandbox/kubernetes_spec.rs | 374 ++++++ .../src/sandbox/kubernetes_warm_pool.rs | 173 +++ crates/alien-infra/src/sandbox/local.rs | 456 ++++++++ crates/alien-infra/src/sandbox/mod.rs | 35 + crates/alien-infra/src/worker/gcp.rs | 63 + .../tests/kubernetes_sandbox_live.rs | 272 +++++ .../alien-k8s-clients/src/kubernetes/mod.rs | 2 + .../src/kubernetes/runtime_classes.rs | 37 + .../src/kubernetes/token_reviews.rs | 144 +++ crates/alien-local/Cargo.toml | 3 + crates/alien-local/src/error.rs | 14 + crates/alien-local/src/lib.rs | 7 + .../src/local_bindings_provider.rs | 45 +- crates/alien-local/src/sandbox_manager.rs | 729 ++++++++++++ crates/alien-local/src/sandbox_route.rs | 606 ++++++++++ crates/alien-local/tests/sandbox_isolation.rs | 466 ++++++++ crates/alien-local/tests/sandbox_route.rs | 315 +++++ crates/alien-manager/src/registry_access.rs | 6 + .../src/routes/registry_proxy.rs | 6 + .../alien-manager/tests/credentials_mint.rs | 7 + crates/alien-operator/src/lib.rs | 24 + crates/alien-operator/src/otlp_server.rs | 10 + packages/core/src/generated/index.ts | 4 + .../schemas/awsSandboxImportData.json | 1 + .../schemas/azureSandboxImportData.json | 1 + .../zod/aws-sandbox-import-data-schema.ts | 20 + .../zod/azure-sandbox-import-data-schema.ts | 17 + packages/core/src/generated/zod/index.ts | 4 + 67 files changed, 11450 insertions(+), 24 deletions(-) create mode 100644 crates/alien-aws-clients/src/aws/lambda_microvms.rs create mode 100644 crates/alien-aws-clients/tests/aws_lambda_microvms_client_tests.rs create mode 100644 crates/alien-azure-clients/src/azure/sandbox_data_plane.rs create mode 100644 crates/alien-azure-clients/src/azure/sandbox_groups.rs create mode 100644 crates/alien-bindings/src/providers/sandbox/agent_protocol.rs create mode 100644 crates/alien-bindings/src/providers/sandbox/aws.rs create mode 100644 crates/alien-bindings/src/providers/sandbox/azure.rs create mode 100644 crates/alien-bindings/src/providers/sandbox/gcp.rs create mode 100644 crates/alien-bindings/src/providers/sandbox/kubernetes.rs create mode 100644 crates/alien-bindings/src/providers/sandbox/local.rs create mode 100644 crates/alien-bindings/src/providers/sandbox/mod.rs create mode 100644 crates/alien-core/src/import/data/aws/sandbox.rs create mode 100644 crates/alien-core/src/import/data/azure/sandbox.rs create mode 100644 crates/alien-infra/src/sandbox/kubernetes.rs create mode 100644 crates/alien-infra/src/sandbox/kubernetes_broker.rs create mode 100644 crates/alien-infra/src/sandbox/kubernetes_eligibility.rs create mode 100644 crates/alien-infra/src/sandbox/kubernetes_route.rs create mode 100644 crates/alien-infra/src/sandbox/kubernetes_spec.rs create mode 100644 crates/alien-infra/src/sandbox/kubernetes_warm_pool.rs create mode 100644 crates/alien-infra/src/sandbox/local.rs create mode 100644 crates/alien-infra/src/sandbox/mod.rs create mode 100644 crates/alien-infra/tests/kubernetes_sandbox_live.rs create mode 100644 crates/alien-k8s-clients/src/kubernetes/runtime_classes.rs create mode 100644 crates/alien-k8s-clients/src/kubernetes/token_reviews.rs create mode 100644 crates/alien-local/src/sandbox_manager.rs create mode 100644 crates/alien-local/src/sandbox_route.rs create mode 100644 crates/alien-local/tests/sandbox_isolation.rs create mode 100644 crates/alien-local/tests/sandbox_route.rs create mode 100644 packages/core/src/generated/schemas/awsSandboxImportData.json create mode 100644 packages/core/src/generated/schemas/azureSandboxImportData.json create mode 100644 packages/core/src/generated/zod/aws-sandbox-import-data-schema.ts create mode 100644 packages/core/src/generated/zod/azure-sandbox-import-data-schema.ts diff --git a/Cargo.lock b/Cargo.lock index 9873cd860..6a128e946 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -714,6 +714,7 @@ dependencies = [ "console_error_panic_hook", "dirs 6.0.0", "dotenvy", + "ed25519-compact", "futures", "futures-util", "getrandom 0.3.4", @@ -787,6 +788,8 @@ dependencies = [ "alien-worker-protocol", "alien-worker-runtime", "async-trait", + "axum 0.8.9", + "base64 0.22.1", "bollard", "bytes", "chrono", diff --git a/crates/alien-aws-clients/src/aws/lambda_microvms.rs b/crates/alien-aws-clients/src/aws/lambda_microvms.rs new file mode 100644 index 000000000..6a99ecf08 --- /dev/null +++ b/crates/alien-aws-clients/src/aws/lambda_microvms.rs @@ -0,0 +1,1019 @@ +//! AWS Lambda MicroVMs client. +//! +//! Hand-rolled rather than SDK-backed: there is no Rust SDK for this API. The wire contract +//! below was read out of `@aws-sdk/client-lambda-microvms`, the published JS SDK, +//! rather than guessed, since an invented path fails as a 404 that looks like a permissions +//! problem. +//! +//! Signed as `lambda`, and the tagging operations sit on Lambda's own `/2017-03-31/tags` path. + +use crate::aws::aws_request_utils::{sign_send_json, AwsSignConfig}; +use crate::aws::credential_provider::AwsCredentialProvider; +use alien_client_core::{ErrorData, Result}; +use alien_error::{AlienError, Context}; +use async_trait::async_trait; +use reqwest::{Client, Method}; +use serde::de::DeserializeOwned; +use serde::{Deserialize, Serialize}; + +#[cfg(feature = "test-utils")] +use mockall::automock; + +/// API version every MicroVM path is prefixed with. +const API_VERSION: &str = "2025-09-09"; + +/// Longest life AWS will mint an endpoint auth token for. +pub const MAX_AUTH_TOKEN_MINUTES: u32 = 60; + +/// A MicroVM image, the Frozen parent of a sandbox's sessions. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct MicrovmImage { + /// Image identifier used in every subsequent path + pub image_identifier: Option, + /// Image ARN + pub image_arn: Option, + /// Current version, which together with the image scopes session discovery + pub image_version: Option, + /// Lifecycle state; an image in CREATING cannot be deleted + pub state: Option, +} + +/// A running MicroVM. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Microvm { + /// MicroVM identifier + pub microvm_id: Option, + /// Per-MicroVM HTTPS endpoint the agent protocol travels over + pub endpoint: Option, + /// Lifecycle state + pub state: Option, + /// Image this MicroVM was started from. + /// + /// The only per-session field that says which sandbox owns it — `RunMicrovm` takes no tags, + /// so without this the answer can only be reached by enumerating the image. + pub image_arn: Option, + /// Version of that image. + pub image_version: Option, +} + +/// Response from listing MicroVMs. +#[derive(Debug, Clone, Default, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ListMicrovmsResponse { + /// The MicroVMs in scope + #[serde(default)] + pub items: Vec, + /// Continuation token; absent when the last page has been read + pub next_token: Option, +} + +/// Response from listing image versions. +#[derive(Debug, Clone, Default, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ListMicrovmImageVersionsResponse { + /// The versions in scope + #[serde(default)] + pub items: Vec, + /// Continuation token + pub next_token: Option, +} + +/// A token authorising requests to one MicroVM's endpoint. +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct MicrovmAuthToken { + /// Header map, not a bearer string. Sending it as `Authorization: Bearer` yields a 403 + /// that reads like a permissions problem rather than a malformed request. + pub auth_token: std::collections::HashMap, +} + +#[cfg_attr(feature = "test-utils", automock)] +#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] +#[cfg_attr(not(target_arch = "wasm32"), async_trait)] +pub trait LambdaMicrovmsApi: Send + Sync + std::fmt::Debug { + /// Reads one image. + async fn get_microvm_image(&self, image_identifier: &str) -> Result; + + /// Deletes an image. Fails while the image is still `CREATING`. + async fn delete_microvm_image(&self, image_identifier: &str) -> Result<()>; + + /// Lists an image's versions. + /// + /// A rolled version stays a cleanup scope until its own MicroVMs are gone, so enumerating + /// only the newest would orphan every session on the previous one. + async fn list_microvm_image_versions( + &self, + image_identifier: &str, + ) -> Result>; + + /// Starts a MicroVM from an image version. + async fn run_microvm( + &self, + image_identifier: &str, + image_version: &str, + client_token: &str, + execution_role_arn: Option, + network_connectors: Vec, + idle_suspend_seconds: Option, + max_lifetime_seconds: Option, + ) -> Result; + + /// Reads one MicroVM. + async fn get_microvm(&self, microvm_id: &str) -> Result; + + /// Terminates a MicroVM. + async fn terminate_microvm(&self, microvm_id: &str) -> Result<()>; + + /// Suspends a MicroVM, preserving its filesystem. + /// + /// Returns once AWS has accepted the request; the MicroVM reaches `SUSPENDED` + /// asynchronously, so a caller that needs the state polls `get_microvm`. + async fn suspend_microvm(&self, microvm_id: &str) -> Result<()>; + + /// Resumes a suspended MicroVM. Also asynchronous. + async fn resume_microvm(&self, microvm_id: &str) -> Result<()>; + + /// Lists every MicroVM started from one image and version, following pagination. + /// + /// Image plus version is the only filter available: `RunMicrovm` takes no `tags`, so a + /// session carries no label of its own to select on. + /// + /// No permission set grants `lambda:ListMicrovms`, because AWS authorizes it against no + /// resource type and the grant could only be account-wide. A caller wiring this up needs to + /// add that grant first, or it will compile and then be refused in a customer account. + async fn list_microvms( + &self, + image_identifier: &str, + image_version: &str, + ) -> Result>; + + /// Mints the short-lived token authorising requests to a MicroVM's endpoint. + /// + /// `expiration_minutes` is required by the API and capped at 60. + async fn create_microvm_auth_token( + &self, + microvm_id: &str, + allowed_ports: Vec, + expiration_minutes: u32, + ) -> Result; +} + +/// Builds the `RunMicrovm` request body. +/// +/// The connector field is `egressNetworkConnectors`, matching +/// `run-microvm --egress-network-connectors`. A MicroVM started without one has public internet +/// access, so getting the name wrong is the opposite of the declared egress policy rather than +/// an error a caller would see. +fn run_microvm_body( + image_identifier: &str, + image_version: &str, + client_token: &str, + execution_role_arn: Option, + egress_network_connectors: Vec, + idle_suspend_seconds: Option, + max_lifetime_seconds: Option, +) -> serde_json::Value { + let mut body = serde_json::json!({ + "imageIdentifier": image_identifier, + "imageVersion": image_version, + // Idempotency: a retried run must not leave a second MicroVM billing quietly. + "clientToken": client_token, + }); + + if let Some(role) = execution_role_arn { + body["executionRoleArn"] = serde_json::Value::String(role); + } + if !egress_network_connectors.is_empty() { + body["egressNetworkConnectors"] = serde_json::json!(egress_network_connectors); + } + if let Some(seconds) = idle_suspend_seconds { + // Suspend only. Auto-resume would bring a session back on a stray request after the + // caller had moved on, which is a bill and a running sandbox nobody is watching. + body["idlePolicy"] = serde_json::json!({ + "maxIdleDurationSeconds": seconds, + "autoResumeEnabled": false, + }); + } + // Lambda terminates the MicroVM when this elapses, so it is the ceiling the declaration asked + // for rather than a hint we would have to police ourselves. + if let Some(seconds) = max_lifetime_seconds { + body["maximumDurationInSeconds"] = serde_json::json!(seconds); + } + body +} + +/// Builds the `CreateMicrovmAuthToken` request body. +/// +/// Each port is an object, not a number — `allowedPorts` is a union of port / range / allPorts, +/// and a bare number array is a different request. `expirationInMinutes` is required. +fn auth_token_body(allowed_ports: Vec, expiration_minutes: u32) -> serde_json::Value { + let ports: Vec = allowed_ports + .into_iter() + .map(|port| serde_json::json!({ "port": port })) + .collect(); + + serde_json::json!({ + "allowedPorts": ports, + "expirationInMinutes": expiration_minutes.min(MAX_AUTH_TOKEN_MINUTES), + }) +} + +/// Client for the Lambda MicroVMs API. +#[derive(Debug, Clone)] +pub struct LambdaMicrovmsClient { + client: Client, + credentials: AwsCredentialProvider, +} + +impl LambdaMicrovmsClient { + /// Builds a client from an HTTP client and credentials. + pub fn new(client: Client, credentials: AwsCredentialProvider) -> Self { + Self { + client, + credentials, + } + } + + fn sign_config(&self) -> AwsSignConfig { + AwsSignConfig { + service_name: "lambda".into(), + region: self.credentials.region().to_string(), + credentials: self.credentials.get_credentials(), + signing_region: None, + } + } + + fn base_url(&self) -> String { + match self.credentials.get_service_endpoint_option("lambda") { + Some(override_url) => override_url.to_string(), + None => format!("https://lambda.{}.amazonaws.com", self.credentials.region()), + } + } + + async fn send( + &self, + method: Method, + path: &str, + query: &[(&str, String)], + body: Option, + operation: &str, + ) -> Result { + self.credentials.ensure_fresh().await?; + + let mut url = format!("{}{path}", self.base_url().trim_end_matches('/')); + if !query.is_empty() { + let encoded: Vec = query + .iter() + .map(|(key, value)| { + format!( + "{key}={}", + form_urlencoded::byte_serialize(value.as_bytes()).collect::() + ) + }) + .collect(); + url.push('?'); + url.push_str(&encoded.join("&")); + } + + let mut builder = self.client.request(method, &url); + if let Some(body) = body { + builder = builder + .header("content-type", "application/json") + .body(serde_json::to_string(&body).map_err(|error| { + AlienError::new(ErrorData::SerializationError { + message: format!("Failed to serialize {operation} body: {error}"), + }) + })?); + } + + match sign_send_json(builder, &self.sign_config()).await { + Ok(value) => Ok(value), + Err(error) => { + let data = classify(&error, operation); + Err(error).context(data) + } + } + } +} + +/// Turns a MicroVMs API failure into the error a caller can act on. +/// +/// Quota exhaustion has to arrive as `QuotaExceeded` — it is retryable and names the limit — and +/// not as a generic failure, because a deployment that hits the account's MicroVM memory quota +/// should back off rather than surface as a broken sandbox. +fn classify(error: &AlienError, operation: &str) -> ErrorData { + // Read structurally and before the rendered-text checks below: a caller has to be able to + // tell "no such MicroVM" from "the call failed", and `GenericError` carries no status to + // recover it from. Matching the body text would classify a 500 whose message happens to + // mention 404 as an absent session. + if let Some(ErrorData::HttpResponseError { http_status, .. }) = &error.error { + if *http_status == 404 { + return ErrorData::RemoteResourceNotFound { + resource_type: "Microvm".to_string(), + resource_name: operation.to_string(), + }; + } + } + + let rendered = format!("{error:?}"); + if rendered.contains("ServiceQuotaExceededException") { + ErrorData::QuotaExceeded { + message: format!( + "Lambda MicroVMs {operation}: account quota exhausted. MicroVM memory is limited \ + per account and Region; retry after existing sandboxes terminate, or request an \ + increase." + ), + } + } else if rendered.contains("ThrottlingException") + || rendered.contains("TooManyRequestsException") + { + ErrorData::RateLimitExceeded { + message: format!("Lambda MicroVMs {operation} was throttled"), + } + } else { + ErrorData::GenericError { + message: format!("Lambda MicroVMs {operation} failed"), + } + } +} + +#[cfg_attr(target_arch = "wasm32", async_trait(?Send))] +#[cfg_attr(not(target_arch = "wasm32"), async_trait)] +impl LambdaMicrovmsApi for LambdaMicrovmsClient { + async fn get_microvm_image(&self, image_identifier: &str) -> Result { + self.send( + Method::GET, + &format!("/{API_VERSION}/microvm-images/{image_identifier}"), + &[], + None, + "GetMicrovmImage", + ) + .await + } + + async fn delete_microvm_image(&self, image_identifier: &str) -> Result<()> { + let _: serde_json::Value = self + .send( + Method::DELETE, + &format!("/{API_VERSION}/microvm-images/{image_identifier}"), + &[], + None, + "DeleteMicrovmImage", + ) + .await?; + Ok(()) + } + + async fn list_microvm_image_versions( + &self, + image_identifier: &str, + ) -> Result> { + let mut versions = Vec::new(); + let mut next: Option = None; + + loop { + let query: Vec<(&str, String)> = next + .as_ref() + .map(|token| vec![("nextToken", token.clone())]) + .unwrap_or_default(); + + let page: ListMicrovmImageVersionsResponse = self + .send( + Method::GET, + &format!("/{API_VERSION}/microvm-images/{image_identifier}/versions"), + &query, + None, + "ListMicrovmImageVersions", + ) + .await?; + + versions.extend(page.items); + next = page.next_token; + if next.is_none() { + return Ok(versions); + } + } + } + + async fn run_microvm( + &self, + image_identifier: &str, + image_version: &str, + client_token: &str, + execution_role_arn: Option, + network_connectors: Vec, + idle_suspend_seconds: Option, + max_lifetime_seconds: Option, + ) -> Result { + let body = run_microvm_body( + image_identifier, + image_version, + client_token, + execution_role_arn, + network_connectors, + idle_suspend_seconds, + max_lifetime_seconds, + ); + + self.send( + Method::POST, + &format!("/{API_VERSION}/microvms"), + &[], + Some(body), + "RunMicrovm", + ) + .await + } + + async fn get_microvm(&self, microvm_id: &str) -> Result { + self.send( + Method::GET, + &format!("/{API_VERSION}/microvms/{microvm_id}"), + &[], + None, + "GetMicrovm", + ) + .await + } + + async fn terminate_microvm(&self, microvm_id: &str) -> Result<()> { + let _: serde_json::Value = self + .send( + Method::DELETE, + &format!("/{API_VERSION}/microvms/{microvm_id}"), + &[], + None, + "TerminateMicrovm", + ) + .await?; + Ok(()) + } + + async fn suspend_microvm(&self, microvm_id: &str) -> Result<()> { + let _: serde_json::Value = self + .send( + Method::POST, + &format!("/{API_VERSION}/microvms/{microvm_id}/suspend"), + &[], + Some(serde_json::json!({})), + "SuspendMicrovm", + ) + .await?; + Ok(()) + } + + async fn resume_microvm(&self, microvm_id: &str) -> Result<()> { + let _: serde_json::Value = self + .send( + Method::POST, + &format!("/{API_VERSION}/microvms/{microvm_id}/resume"), + &[], + Some(serde_json::json!({})), + "ResumeMicrovm", + ) + .await?; + Ok(()) + } + + async fn list_microvms( + &self, + image_identifier: &str, + image_version: &str, + ) -> Result> { + let mut microvms = Vec::new(); + let mut next: Option = None; + + loop { + let mut query = vec![ + ("imageIdentifier", image_identifier.to_string()), + ("imageVersion", image_version.to_string()), + ]; + if let Some(token) = next.as_ref() { + query.push(("nextToken", token.clone())); + } + + let page: ListMicrovmsResponse = self + .send( + Method::GET, + &format!("/{API_VERSION}/microvms"), + &query, + None, + "ListMicrovms", + ) + .await?; + + microvms.extend(page.items); + next = page.next_token; + // Paginate to exhaustion: a partial list during teardown silently orphans whatever + // sat on the pages nobody read. + if next.is_none() { + return Ok(microvms); + } + } + } + + async fn create_microvm_auth_token( + &self, + microvm_id: &str, + allowed_ports: Vec, + expiration_minutes: u32, + ) -> Result { + self.send( + Method::POST, + &format!("/{API_VERSION}/microvms/{microvm_id}/auth-token"), + &[], + Some(auth_token_body(allowed_ports, expiration_minutes)), + "CreateMicrovmAuthToken", + ) + .await + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The declared ceiling has to reach the wire: Lambda is what terminates the MicroVM when it + /// elapses, so a value we accept and drop would leave a sandbox running past a limit its + /// stack declared — the shape of claim this resource exists to keep. + #[test] + fn a_declared_lifetime_reaches_the_run_request() { + let body = run_microvm_body("img", "1", "token", None, vec![], None, Some(3600)); + assert_eq!(body["maximumDurationInSeconds"], serde_json::json!(3600)); + + let unbounded = run_microvm_body("img", "1", "token", None, vec![], None, None); + assert!( + unbounded.get("maximumDurationInSeconds").is_none(), + "an undeclared ceiling stays absent rather than becoming a made-up one" + ); + } + + /// Built the way the transport builds it, not by hand: `handle_json_response` raises + /// `HttpResponseError`, whose variant declares no status of its own, so anything reading + /// `http_status_code` sees the derive's 500 default and never sees the 404. A caller has to be + /// able to tell an absent MicroVM from a failed call, so the status is read structurally here + /// and re-raised as the variant that carries it. + #[test] + fn an_absent_microvm_is_classified_as_not_found() { + let response = AlienError::new(ErrorData::HttpResponseError { + message: "GetMicrovm failed".to_string(), + url: "https://lambda.example.invalid/microvms/mv-1".to_string(), + http_status: 404, + http_request_text: None, + http_response_text: Some("{\"message\":\"Microvm not found\"}".to_string()), + }); + + assert!( + matches!( + classify(&response, "GetMicrovm"), + ErrorData::RemoteResourceNotFound { .. } + ), + "a 404 has to survive as something the caller can match on" + ); + + // The trap this replaced: the status never reaches the wrapped error. + assert_ne!( + response.http_status_code, + Some(404), + "if this ever becomes Some(404), the status is readable directly and the arm above \ + can be simplified" + ); + } + + /// A body that merely mentions 404 is not an absent MicroVM. Classifying on rendered text + /// would turn a throttle or a server error into "the session is gone", which starts a second + /// sandbox while the first keeps running. + #[test] + fn only_the_status_makes_a_microvm_absent() { + for status in [429, 500, 503] { + let error = AlienError::new(ErrorData::HttpResponseError { + message: "GetMicrovm failed".to_string(), + url: "https://lambda.example.invalid/microvms/404".to_string(), + http_status: status, + http_request_text: None, + http_response_text: Some("upstream said 404 somewhere".to_string()), + }); + + assert!( + !matches!( + classify(&error, "GetMicrovm"), + ErrorData::RemoteResourceNotFound { .. } + ), + "{status} is not an absent MicroVM" + ); + } + } + + /// Paths were read out of the published JS SDK, not guessed. Pinning them here + /// means a future edit that mistypes one fails at build time rather than as a 404 that + /// reads like a permissions error. + #[test] + fn the_api_version_matches_the_published_wire_contract() { + assert_eq!(API_VERSION, "2025-09-09"); + } + + /// The egress connector is the whole of what makes `egress: deny` real, and the failure + /// mode of naming its field wrong is a MicroVM with public internet access rather than a + /// rejected request. AWS spells it `--egress-network-connectors` on `run-microvm`. + #[test] + fn a_session_carries_its_egress_connector_under_the_name_aws_reads() { + let body = run_microvm_body( + "image", + "3", + "token", + None, + vec!["arn:aws:lambda:us-west-2:123456789012:network-connector:sbx".to_string()], + None, + None, + ); + + assert_eq!( + body["egressNetworkConnectors"], + serde_json::json!(["arn:aws:lambda:us-west-2:123456789012:network-connector:sbx"]) + ); + assert!( + body.get("networkConnectors").is_none(), + "the connector must not also ride a field RunMicrovm does not read: {body}" + ); + } + + /// Both fields were wrong in the first cut: ports encoded as bare numbers, and + /// `expirationInMinutes` omitted entirely though the API requires it. Neither would have + /// failed until a live call, so the shape that works is pinned here. + #[test] + fn the_auth_token_request_encodes_ports_as_objects() { + let body = auth_token_body(vec![8971], 30); + + assert_eq!( + body["allowedPorts"], + serde_json::json!([{ "port": 8971 }]), + "allowedPorts is a union of port/range/allPorts, not a list of numbers" + ); + assert_eq!(body["expirationInMinutes"], 30); + } + + /// AWS caps the token at 60 minutes; asking for more is a rejected request, so the clamp + /// happens here rather than arriving as a validation error at the call site. + #[test] + fn an_over_long_expiry_is_clamped_to_the_documented_maximum() { + assert_eq!(auth_token_body(vec![80], 600)["expirationInMinutes"], 60); + } + + /// Quota exhaustion must arrive as a typed, retryable error naming the + /// quota. It is the one failure a caller should back off from rather than treat as a broken + /// sandbox, and a generic error gives them nothing to branch on. + #[test] + fn quota_exhaustion_is_typed_and_retryable() { + let raw = AlienError::new(ErrorData::GenericError { + message: "ServiceQuotaExceededException: memory limit".to_string(), + }); + + let data = classify(&raw, "RunMicrovm"); + let error = AlienError::new(data); + + assert_eq!(error.code, "QUOTA_EXCEEDED"); + assert!(error.retryable, "a quota clears when sandboxes terminate"); + assert!( + error.to_string().contains("quota exhausted"), + "the message must name the condition: {error}" + ); + } + + #[test] + fn throttling_is_distinguished_from_quota() { + let raw = AlienError::new(ErrorData::GenericError { + message: "ThrottlingException: slow down".to_string(), + }); + + assert_eq!(AlienError::new(classify(&raw, "RunMicrovm")).code, "RATE_LIMIT_EXCEEDED"); + } + + /// Anything unrecognised stays generic rather than being guessed into a typed error a + /// caller would then branch on incorrectly. + #[test] + fn an_unrecognised_failure_is_not_invented_into_a_typed_error() { + let raw = AlienError::new(ErrorData::GenericError { + message: "ValidationException: bad ARN".to_string(), + }); + + assert_eq!(AlienError::new(classify(&raw, "RunMicrovm")).code, "GENERIC_ERROR"); + } + + #[test] + fn an_auth_token_is_a_header_map_not_a_bearer_string() { + let token: MicrovmAuthToken = serde_json::from_str( + r#"{"authToken":{"X-aws-proxy-auth":"eyJ...","X-aws-proxy-port":"8080"}}"#, + ) + .expect("deserializes"); + + assert_eq!(token.auth_token.len(), 2); + assert!(token.auth_token.contains_key("X-aws-proxy-auth")); + } + + /// Reading a response key that does not exist reports a working call as a failure. + /// Listing keys off `items` is the shape the API actually returns. + #[test] + fn a_list_response_reads_items_and_a_next_token() { + let page: ListMicrovmsResponse = serde_json::from_str( + r#"{"items":[{"microvmId":"microvm-1","endpoint":"abc.lambda-url","state":"RUNNING"}],"nextToken":"t2"}"#, + ) + .expect("deserializes"); + + assert_eq!(page.items.len(), 1); + assert_eq!(page.items[0].microvm_id.as_deref(), Some("microvm-1")); + assert_eq!(page.next_token.as_deref(), Some("t2")); + } + + #[test] + fn an_empty_list_response_is_not_an_error() { + let page: ListMicrovmsResponse = + serde_json::from_str("{}").expect("an absent items key means none, not a parse error"); + assert!(page.items.is_empty()); + assert!(page.next_token.is_none()); + } +} + +/// Calls `CreateMicrovmImage` directly, so the service's own error is visible. +/// +/// CloudControl reports a failed image build as `NotStabilized` with no `StateReason` and no log +/// group, which says nothing about why. This goes straight at the API. Lives here rather than in +/// `tests/` because `send` is private, and adding a production `create_microvm_image` for a +/// diagnostic would be a method nothing else calls — setup builds images through IaC. +#[cfg(test)] +mod live_image_create { + use super::*; + use crate::aws::aws_request_utils::AwsRequestSigner; + use crate::{AwsCredentialProvider, AwsCredentials}; + use std::path::PathBuf as StdPathBuf; + + fn client() -> LambdaMicrovmsClient { + let root: StdPathBuf = workspace_root::get_workspace_root(); + dotenvy::from_path(root.join(".env.test")).ok(); + + let config = crate::AwsClientConfig { + account_id: std::env::var("AWS_TARGET_ACCOUNT_ID").expect("AWS_TARGET_ACCOUNT_ID"), + region: std::env::var("AWS_TARGET_REGION").expect("AWS_TARGET_REGION"), + credentials: AwsCredentials::AccessKeys { + access_key_id: std::env::var("AWS_TARGET_ACCESS_KEY_ID") + .expect("AWS_TARGET_ACCESS_KEY_ID"), + secret_access_key: std::env::var("AWS_TARGET_SECRET_ACCESS_KEY") + .expect("AWS_TARGET_SECRET_ACCESS_KEY"), + session_token: None, + }, + service_overrides: None, + }; + + LambdaMicrovmsClient::new(Client::new(), AwsCredentialProvider::from_config_sync(config)) + } + + /// Deletes an image the way the API wants it, versions first. + /// + /// CloudControl accepts a delete on a `CREATED` image and never removes it — the versions + /// hold it. `DeleteMicrovmImageVersion` is in `sandbox/provision` for exactly this reason. + #[tokio::test] + #[ignore] + async fn delete_images_versions_first() { + let client = client(); + for name in std::env::var("PROBE_IMAGES").expect("PROBE_IMAGES").split(',') { + let name = name.trim(); + if name.is_empty() { + continue; + } + let versions = client.list_microvm_image_versions(name).await; + match &versions { + Ok(list) => println!("{name}: {} version(s)", list.len()), + Err(error) => println!("{name}: list refused: {error}"), + } + for version in versions.unwrap_or_default() { + let Some(v) = version.image_version.as_deref() else { continue }; + let path = format!("/{API_VERSION}/microvm-images/{name}/versions/{v}"); + let result: std::result::Result = client + .send(Method::DELETE, &path, &[], None, "DeleteMicrovmImageVersion") + .await; + println!(" version {v}: {}", if result.is_ok() { "deleted" } else { "refused" }); + } + match client.delete_microvm_image(name).await { + Ok(()) => println!(" image deleted"), + Err(error) => println!(" image refused: {error}"), + } + } + } + + #[tokio::test] + #[ignore] + async fn create_an_image_and_report_the_services_own_error() { + let name = std::env::var("PROBE_IMAGE_NAME").expect("PROBE_IMAGE_NAME"); + let build_role = std::env::var("PROBE_BUILD_ROLE_ARN").expect("PROBE_BUILD_ROLE_ARN"); + let artifact = std::env::var("PROBE_ARTIFACT_URI").expect("PROBE_ARTIFACT_URI"); + let region = std::env::var("AWS_TARGET_REGION").expect("AWS_TARGET_REGION"); + + let mut body = serde_json::json!({ + "name": name, + "baseImageArn": format!("arn:aws:lambda:{region}:aws:microvm-image:al2023-1"), + "buildRoleArn": build_role, + "codeArtifact": { "uri": artifact }, + }); + for extra in std::env::var("PROBE_EXTRA").unwrap_or_default().split(',') { + match extra.trim() { + "hooks" => body["hooks"] = serde_json::json!({ + "port": 8971, + "microvmImageHooks": { "ready": "ENABLED", "readyTimeoutInSeconds": 120 }, + "microvmHooks": { "run": "ENABLED", "runTimeoutInSeconds": 30, + "resume": "ENABLED", "resumeTimeoutInSeconds": 30 } + }), + "env" => body["environmentVariables"] = serde_json::json!({ + "ALIEN_SANDBOX_ROOT": "/sandbox", + "ALIEN_SANDBOX_PORT": "8971", + "ALIEN_SANDBOX_AUTHORIZATION": "transport", + "ALIEN_SANDBOX_EXEC_UID": "60000", + "ALIEN_SANDBOX_EXEC_GID": "60000" + }), + "cpu" => body["cpuConfigurations"] = serde_json::json!([{ "architecture": "ARM_64" }]), + "res" => body["resources"] = serde_json::json!([{ "minimumMemoryInMiB": 512 }]), + "log" => body["logging"] = serde_json::json!({ "cloudWatch": { "logGroup": std::env::var("PROBE_LOG_GROUP").unwrap_or_else(|_| "/aws/lambda-microvms/alienden".to_string()) } }), + "conn" => if let Some(c) = std::env::var("PROBE_CONNECTOR_ARN").ok() { + body["egressNetworkConnectors"] = serde_json::json!([c]); + }, + _ => {} + } + } + println!("request: {}", serde_json::to_string_pretty(&body).unwrap()); + + // Signed by hand rather than through `send`, which drops the response body — and the + // body is where AWS puts the reason a 400 happened. + let client = client(); + client.credentials.ensure_fresh().await.expect("credentials"); + let url = format!("{}/{API_VERSION}/microvm-images", client.base_url()); + let signed = client + .client + .request(Method::POST, &url) + .header("content-type", "application/json") + .body(serde_json::to_string(&body).unwrap()) + .sign_aws_request(&client.sign_config()) + .expect("signing"); + + let response = signed.send().await.expect("the request should reach AWS"); + let status = response.status(); + let text = response.text().await.unwrap_or_default(); + println!("STATUS: {status}"); + println!("BODY: {text}"); + } +} + +/// A MicroVM under `deny` cannot reach the internet. +/// +/// Two MicroVMs from one image — one started with the egress connector the emitted packages +/// render, one +/// with none — each asked to reach a public address through the agent's own exec path. The +/// comparison is the point: a `deny` MicroVM that cannot reach the internet proves nothing if the +/// control cannot either. +#[cfg(test)] +mod live_deny { + use super::*; + use crate::{AwsCredentialProvider, AwsCredentials}; + use std::path::PathBuf as StdPathBuf; + use std::time::Duration; + + fn client() -> LambdaMicrovmsClient { + let root: StdPathBuf = workspace_root::get_workspace_root(); + dotenvy::from_path(root.join(".env.test")).ok(); + let config = crate::AwsClientConfig { + account_id: std::env::var("AWS_TARGET_ACCOUNT_ID").expect("AWS_TARGET_ACCOUNT_ID"), + region: std::env::var("AWS_TARGET_REGION").expect("AWS_TARGET_REGION"), + credentials: AwsCredentials::AccessKeys { + access_key_id: std::env::var("AWS_TARGET_ACCESS_KEY_ID").expect("key"), + secret_access_key: std::env::var("AWS_TARGET_SECRET_ACCESS_KEY").expect("secret"), + session_token: None, + }, + service_overrides: None, + }; + LambdaMicrovmsClient::new(Client::new(), AwsCredentialProvider::from_config_sync(config)) + } + + /// Runs one command through the agent and returns its raw NDJSON body. + async fn exec(client: &LambdaMicrovmsClient, id: &str, endpoint: &str, command: &[&str]) -> String { + let token = client + .create_microvm_auth_token(id, vec![8971], 10) + .await + .expect("CreateMicrovmAuthToken"); + let mut request = Client::new() + .post(format!("https://{endpoint}/v1/exec")) + .header("X-aws-proxy-port", "8971") + .json(&serde_json::json!({"command": command, "deadlineMs": 15000})); + for (name, value) in token.auth_token { + request = request.header(name, value); + } + let body = match request.send().await { + Ok(response) => response.text().await.unwrap_or_default(), + Err(error) => return format!(""), + }; + + // The agent streams NDJSON frames whose `data` is base64, so a plain string match against + // the body would test the encoding rather than what the command printed. + use base64::Engine as _; + let mut decoded = String::new(); + for line in body.lines() { + let Ok(frame) = serde_json::from_str::(line) else { + continue; + }; + if let Some(data) = frame.get("data").and_then(|d| d.as_str()) { + if let Ok(bytes) = base64::engine::general_purpose::STANDARD.decode(data) { + decoded.push_str(&String::from_utf8_lossy(&bytes)); + } + } + if let Some(code) = frame.get("code") { + decoded.push_str(&format!("\nexit={code}")); + } + } + decoded + } + + async fn start(client: &LambdaMicrovmsClient, image: &str, version: &str, connectors: Vec) -> (String, String) { + let token = uuid::Uuid::new_v4().simple().to_string(); + let microvm = client + .run_microvm(image, version, &token, None, connectors, None, None) + .await + .expect("RunMicrovm"); + let id = microvm.microvm_id.expect("a MicroVM id"); + for _ in 0..60 { + let current = client.get_microvm(&id).await.expect("get_microvm"); + if current.state.as_deref() == Some("RUNNING") { + return (id, current.endpoint.expect("an endpoint")); + } + tokio::time::sleep(Duration::from_secs(5)).await; + } + panic!("MicroVM {id} never reached RUNNING"); + } + + /// Which form of `imageIdentifier` each call accepts. + /// + /// `RunMicrovm` refuses a bare name with "Malformed ARN - doesn't start with 'arn:'", and the + /// import data feeds both this and `GetMicrovmImage`, so the two have to agree. + #[tokio::test] + #[ignore] + async fn which_image_identifier_form_each_call_accepts() { + let name = std::env::var("PROBE_IMAGE_NAME").expect("PROBE_IMAGE_NAME"); + let arn = std::env::var("PROBE_IMAGE_ARN").expect("PROBE_IMAGE_ARN"); + let client = client(); + + for (label, identifier) in [("name", &name), ("arn", &arn)] { + let got = client.get_microvm_image(identifier).await; + println!("GetMicrovmImage({label}) -> {}", if got.is_ok() { "ok" } else { "REFUSED" }); + + let token = uuid::Uuid::new_v4().simple().to_string(); + let ran = client.run_microvm(identifier, "1.0", &token, None, Vec::new(), None, None) + .await; + match ran { + Ok(microvm) => { + println!("RunMicrovm({label}) -> ok"); + if let Some(id) = microvm.microvm_id { + let _ = client.terminate_microvm(&id).await; + } + } + Err(error) => println!("RunMicrovm({label}) -> REFUSED: {}", format!("{error:?}") + .split("http_response_text").nth(1).unwrap_or("").chars().take(90).collect::()), + } + } + } + + #[tokio::test] + #[ignore] + async fn a_denied_sandbox_cannot_reach_the_internet_and_an_open_one_can() { + let image = std::env::var("PROBE_IMAGE_NAME").expect("PROBE_IMAGE_NAME"); + let version = std::env::var("PROBE_IMAGE_VERSION").unwrap_or_else(|_| "1.0".to_string()); + let connector = std::env::var("PROBE_CONNECTOR_ARN").expect("PROBE_CONNECTOR_ARN"); + let client = client(); + + let probe = ["/usr/bin/curl", "-sS", "--max-time", "8", "-o", "/dev/null", + "-w", "HTTP:%{http_code}", "https://example.com"]; + + let (denied, denied_endpoint) = start(&client, &image, &version, vec![connector]).await; + println!("DENY microvm={denied}"); + let denied_output = exec(&client, &denied, &denied_endpoint, &probe).await; + println!("DENY OUTPUT: {denied_output}"); + let _ = client.terminate_microvm(&denied).await; + + let (open, open_endpoint) = start(&client, &image, &version, Vec::new()).await; + println!("CONTROL microvm={open}"); + let open_output = exec(&client, &open, &open_endpoint, &probe).await; + println!("CONTROL OUTPUT: {open_output}"); + let _ = client.terminate_microvm(&open).await; + + // The control first. A `deny` MicroVM that cannot reach the internet proves nothing if + // the image, the agent or the probe was broken for both. + assert!( + open_output.contains("HTTP:200"), + "the control must reach the internet, or the deny result means nothing:\n{open_output}" + ); + assert!( + !denied_output.contains("HTTP:200"), + "a sandbox under deny reached the internet:\n{denied_output}" + ); + assert!( + denied_output.contains("HTTP:000"), + "deny should fail to connect rather than get some other status:\n{denied_output}" + ); + } +} diff --git a/crates/alien-aws-clients/src/aws/mod.rs b/crates/alien-aws-clients/src/aws/mod.rs index dfd56176b..481bfee9c 100644 --- a/crates/alien-aws-clients/src/aws/mod.rs +++ b/crates/alien-aws-clients/src/aws/mod.rs @@ -61,6 +61,7 @@ pub mod elbv2; pub mod eventbridge; pub mod iam; pub mod lambda; +pub mod lambda_microvms; pub mod rds; pub mod resourcegroupstagging; pub mod s3; diff --git a/crates/alien-aws-clients/tests/aws_lambda_microvms_client_tests.rs b/crates/alien-aws-clients/tests/aws_lambda_microvms_client_tests.rs new file mode 100644 index 000000000..04e3e06f3 --- /dev/null +++ b/crates/alien-aws-clients/tests/aws_lambda_microvms_client_tests.rs @@ -0,0 +1,248 @@ +//! Live lifecycle for Lambda MicroVMs, driven through the client Alien actually ships. +//! +//! `#[ignore]` because it builds a real MicroVM image (~160s) and runs a MicroVM. Run with +//! `--ignored` against an AWS account configured in `.env.test` (see `AWS_TARGET_*`). +//! +//! Every resource this creates is torn down at the end, and the image delete waits for a +//! terminal state first: `DeleteMicrovmImage` fails while the image is still `CREATING`, which +//! is how one gets leaked. + +use std::path::PathBuf as StdPathBuf; +use std::time::Duration; + +use alien_aws_clients::aws::lambda_microvms::{LambdaMicrovmsApi, LambdaMicrovmsClient}; +use alien_aws_clients::{AwsCredentialProvider, AwsCredentials}; +use reqwest::Client; + +/// Slot-scoped so two runs cannot collide on a name. +fn slot() -> String { + std::env::var("ALIEN_E2E_SLOT").unwrap_or_else(|_| "00".to_string()) +} + +fn client() -> LambdaMicrovmsClient { + let root: StdPathBuf = workspace_root::get_workspace_root(); + dotenvy::from_path(root.join(".env.test")).ok(); + + let config = alien_aws_clients::AwsClientConfig { + account_id: std::env::var("AWS_TARGET_ACCOUNT_ID").expect("AWS_TARGET_ACCOUNT_ID"), + region: std::env::var("AWS_TARGET_REGION").expect("AWS_TARGET_REGION"), + credentials: AwsCredentials::AccessKeys { + access_key_id: std::env::var("AWS_TARGET_ACCESS_KEY_ID") + .expect("AWS_TARGET_ACCESS_KEY_ID"), + secret_access_key: std::env::var("AWS_TARGET_SECRET_ACCESS_KEY") + .expect("AWS_TARGET_SECRET_ACCESS_KEY"), + session_token: None, + }, + service_overrides: None, + }; + + LambdaMicrovmsClient::new(Client::new(), AwsCredentialProvider::from_config_sync(config)) +} + +/// Read-only reachability: the paths, the signing name and the response shape. +/// +/// Separate from the lifecycle test because it creates nothing, so it can run whenever without +/// leaving anything to clean up. +#[tokio::test] +#[ignore] +async fn the_microvms_api_is_reachable_and_lists_nothing_unexpected() { + let client = client(); + + let error = client + .list_microvm_image_versions("does-not-exist") + .await + .expect_err("listing versions of a missing image must fail"); + + // Printed in full: the status alone cannot distinguish "no such image" from "not + // authorized", and the body is the only thing that says which. + println!("list_microvm_image_versions(missing) -> {error:?}"); + + let rendered = format!("{error:?}"); + assert!( + !rendered.contains("403") && !rendered.to_lowercase().contains("not authorized"), + "the credentials should reach the API, not be refused by it: {rendered}" + ); +} + +/// The full lifecycle, if `ALIEN_SANDBOX_TEST_IMAGE_ARN` names an image that already exists. +/// +/// Building an image needs an S3 bundle and a build role, which are provisioned outside this +/// test; pointing at a prepared image keeps the run to the part the client owns — run, reach, +/// terminate — and keeps a failure from stranding a half-built image. +#[tokio::test] +#[ignore] +async fn a_microvm_runs_serves_the_agent_and_terminates() { + let Ok(image_arn) = std::env::var("ALIEN_SANDBOX_TEST_IMAGE_ARN") else { + eprintln!("ALIEN_SANDBOX_TEST_IMAGE_ARN not set; skipping the lifecycle"); + return; + }; + let image_version = + std::env::var("ALIEN_SANDBOX_TEST_IMAGE_VERSION").unwrap_or_else(|_| "1".to_string()); + let client = client(); + + let microvm = client + .run_microvm( + &image_arn, + &image_version, + &format!("alien-sbx-{}-{}", slot(), std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap().as_secs()), + None, + Vec::new(), + None, + // A live MicroVM that outlives a failed test run still bills, so the live suite caps + // it even though the unit paths leave it undeclared. + Some(600), + ) + .await + .expect("RunMicrovm should start a MicroVM"); + + let microvm_id = microvm.microvm_id.clone().expect("a MicroVM id"); + println!("started {microvm_id}"); + + // Terminate whatever happens next: a panic between here and the end would otherwise leave a + // MicroVM running and billing. + let outcome = tokio::time::timeout( + Duration::from_secs(300), + exercise(&client, µvm_id), + ) + .await; + + client + .terminate_microvm(µvm_id) + .await + .expect("terminate should succeed"); + + let terminated = client + .get_microvm(µvm_id) + .await + .expect("the MicroVM should still be readable after terminate"); + println!("after terminate: state={:?}", terminated.state); + + outcome.expect("the lifecycle should finish inside its deadline"); +} + +/// Where the image places the agent. Duplicated rather than imported: this crate is the cloud +/// client and must not depend on the build crate to run one test. +const AGENT_PATH: &str = "/usr/local/bin/alien-sandbox-agent"; + +/// Runs one command through the agent and returns its raw NDJSON body. +async fn exec_in( + client: &LambdaMicrovmsClient, + microvm_id: &str, + endpoint: &str, + command: &[&str], +) -> String { + let token = client + .create_microvm_auth_token(microvm_id, vec![8971], 10) + .await + .expect("CreateMicrovmAuthToken"); + + let mut request = Client::new() + .post(format!("https://{endpoint}/v1/exec")) + .header("X-aws-proxy-port", "8971") + .json(&serde_json::json!({"command": command, "deadlineMs": 10000})); + for (name, value) in token.auth_token { + request = request.header(name, value); + } + + request + .send() + .await + .expect("exec responds") + .text() + .await + .unwrap_or_default() +} + +/// Waits for the MicroVM to serve, then proves the agent inside it answers. +async fn exercise(client: &LambdaMicrovmsClient, microvm_id: &str) { + let mut endpoint = None; + for _ in 0..60 { + let microvm = client.get_microvm(microvm_id).await.expect("get_microvm"); + if microvm.state.as_deref() == Some("RUNNING") { + endpoint = microvm.endpoint; + break; + } + tokio::time::sleep(Duration::from_secs(5)).await; + } + let endpoint = endpoint.expect("the MicroVM should reach RUNNING and report an endpoint"); + + let token = client + .create_microvm_auth_token(microvm_id, vec![8971], 10) + .await + .expect("CreateMicrovmAuthToken"); + + let mut request = Client::new() + .get(format!("https://{endpoint}/v1/health")) + .header("X-aws-proxy-port", "8971"); + for (name, value) in token.auth_token { + request = request.header(name, value); + } + + let response = request.send().await.expect("the endpoint should answer"); + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + println!("GET /v1/health -> {status} {body}"); + + assert!(status.is_success(), "the agent should serve health: {status} {body}"); + assert!( + body.contains("protocolVersion"), + "the agent should report its protocol version, got: {body}" + ); + + // The boundary that matters, proven where it actually has to hold: a command run inside a + // real MicroVM must come back as the unprivileged uid, not as the agent. + let frames = exec_in(client, microvm_id, &endpoint, &["/usr/bin/id"]).await; + println!("POST /v1/exec ->\n{frames}"); + + let decoded: String = frames + .lines() + .filter_map(|line| serde_json::from_str::(line).ok()) + .filter_map(|frame| frame["data"].as_str().map(str::to_string)) + .filter_map(|data| { + use base64::Engine as _; + base64::engine::general_purpose::STANDARD.decode(data).ok() + }) + .filter_map(|bytes| String::from_utf8(bytes).ok()) + .collect(); + + assert!( + decoded.contains("uid=60000"), + "a command in a real MicroVM must run as the unprivileged uid, got: {decoded}" + ); + + // The other half of the same boundary: the uid drop is only worth having if the workload + // cannot rewrite the supervisor that performs it. The image owns the agent as root; this is + // the check that the running guest agrees. + let frames = exec_in(client, microvm_id, &endpoint, &["/usr/bin/touch", AGENT_PATH]).await; + println!("touch {AGENT_PATH} ->\n{frames}"); + + let exit_code = frames + .lines() + .filter_map(|line| serde_json::from_str::(line).ok()) + .find_map(|frame| frame["code"].as_i64()); + + assert_eq!( + exit_code, + Some(1), + "the exec uid must not be able to write the agent binary, got: {frames}" + ); +} + +/// Deletes an image named by `ALIEN_SANDBOX_TEST_IMAGE_ARN`. Cleanup for a supervised run, and +/// it exercises `delete_microvm_image` — which is what teardown depends on. +#[tokio::test] +#[ignore] +async fn delete_the_named_image() { + let Ok(arn) = std::env::var("ALIEN_SANDBOX_TEST_IMAGE_ARN") else { + return; + }; + let client = client(); + + let image = client.get_microvm_image(&arn).await.expect("reads the image"); + println!("before delete: state={:?}", image.state); + + client.delete_microvm_image(&arn).await.expect("delete should succeed"); + + let after = client.get_microvm_image(&arn).await; + println!("after delete: {after:?}"); +} diff --git a/crates/alien-azure-clients/src/azure/mod.rs b/crates/alien-azure-clients/src/azure/mod.rs index b252da790..907d97a98 100644 --- a/crates/alien-azure-clients/src/azure/mod.rs +++ b/crates/alien-azure-clients/src/azure/mod.rs @@ -14,6 +14,8 @@ pub mod authorization; pub mod blob_containers; pub mod cognitive_services; pub mod common; +pub mod sandbox_data_plane; +pub mod sandbox_groups; pub mod compute; pub mod container_apps; pub mod containerregistry; diff --git a/crates/alien-azure-clients/src/azure/sandbox_data_plane.rs b/crates/alien-azure-clients/src/azure/sandbox_data_plane.rs new file mode 100644 index 000000000..d39516abd --- /dev/null +++ b/crates/alien-azure-clients/src/azure/sandbox_data_plane.rs @@ -0,0 +1,333 @@ +//! Azure Container Apps Sandboxes — the ADC data plane. +//! +//! A **second endpoint** from ARM, at `management..azuredevcompute.io`, gated by the +//! `Container Apps SandboxGroup Data Owner` role. Subscription Owner returns 403 here, so +//! management permissions alone provision a group cleanly and then fail at first exec. +//! +//! Microsoft's published data-plane REST reference covers `sessionPools` only, so the contract +//! below was read out of the `azure-containerapps-sandbox` PyPI package (0.1.0b4) rather than +//! guessed. That package is a preview whose surface Microsoft says may change, so the paths are +//! pinned here with tests and re-read on upgrade rather than assumed stable. + +use crate::azure::common::{AzureClientBase, AzureRequestBuilder}; +use crate::azure::token_cache::AzureTokenCache; +use alien_client_core::{ErrorData, Result}; +use alien_error::{Context, IntoAlienError}; +use async_trait::async_trait; +use reqwest::Method; +use serde::{Deserialize, Serialize}; + +#[cfg(feature = "test-utils")] +use mockall::automock; + +/// Data-plane API version, from the SDK's `ApiVersion.V2026_02_01_PREVIEW`. +pub const API_VERSION: &str = "2026-02-01-preview"; + +/// Scope the data plane is signed for. Distinct from ARM's, which is why a token minted for +/// `management.azure.com` fails here in a way that looks like a permissions problem. +const ADC_SCOPE: &str = "https://management.azuredevcompute.io/.default"; + +/// A sandbox as the data plane reports it. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Sandbox { + /// Sandbox id within its group + pub id: String, + /// `Running` or `Stopped` + #[serde(skip_serializing_if = "Option::is_none")] + pub status: Option, +} + +/// Result of a shell command. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ExecResult { + /// Captured stdout + #[serde(default)] + pub stdout: String, + /// Captured stderr + #[serde(default)] + pub stderr: String, + /// Process exit code, absent when the service did not report one + #[serde(default)] + pub exit_code: Option, +} + +#[cfg_attr(feature = "test-utils", automock)] +#[async_trait] +pub trait SandboxDataPlaneApi: Send + Sync + std::fmt::Debug { + /// Creates a sandbox from a disk image. + async fn create_sandbox(&self, group: &str, disk: &str, cpu: &str, memory: &str) + -> Result; + + /// Reads a sandbox. A 404 is how deletion is confirmed. + async fn get_sandbox(&self, group: &str, sandbox_id: &str) -> Result; + + /// Deletes a sandbox. Returns before it is gone; confirm by polling `get_sandbox` to 404. + async fn delete_sandbox(&self, group: &str, sandbox_id: &str) -> Result<()>; + + /// Runs a shell command inside a sandbox. + /// + /// The body the SDK sends is `command` plus an optional `workingDirectory`, and nothing else + /// — there is no timeout field, so a wall-clock ceiling can only be applied by the caller. + async fn execute_shell_command( + &self, + group: &str, + sandbox_id: &str, + command: &str, + working_directory: Option, + ) -> Result; +} + +/// The `executeShellCommand` body, which is `command` plus an optional `workingDirectory` and +/// nothing else — read out of the preview SDK, which sends exactly these two. +fn exec_body(command: &str, working_directory: Option) -> serde_json::Value { + let mut payload = serde_json::json!({ "command": command }); + if let Some(directory) = working_directory { + payload["workingDirectory"] = serde_json::Value::String(directory); + } + payload +} + +/// Client for the ADC sandbox data plane. +#[derive(Debug)] +pub struct AzureSandboxDataPlaneClient { + base: AzureClientBase, + token_cache: AzureTokenCache, + resource_group: String, +} + +impl AzureSandboxDataPlaneClient { + /// Builds a client against the region's ADC endpoint. + pub fn new( + client: reqwest::Client, + region: &str, + resource_group: &str, + token_cache: AzureTokenCache, + ) -> Self { + let endpoint = format!("https://management.{region}.azuredevcompute.io"); + + Self { + base: AzureClientBase::with_client_config( + client, + endpoint, + token_cache.config().clone(), + ), + token_cache, + resource_group: resource_group.to_string(), + } + } + + /// Path prefix scoping every call to one sandbox group. + /// + /// Note it is **not** an ARM path: there is no `providers/Microsoft.App` segment. + fn group_path(&self, group: &str) -> String { + format!( + "/subscriptions/{}/resourceGroups/{}/sandboxGroups/{group}", + self.token_cache.config().subscription_id, + self.resource_group + ) + } + + fn sandbox_path(&self, group: &str, sandbox_id: &str) -> String { + format!("{}/sandboxes/{sandbox_id}", self.group_path(group)) + } + + async fn parse( + response: reqwest::Response, + operation: &str, + ) -> Result { + // The status is carried structurally rather than left for a caller to find in the + // message: a body can contain "404" in a path or a trace id, and classifying on the + // rendered text turns an unrelated failure into "the session is gone". + let status = response.status(); + let url = response.url().to_string(); + if !status.is_success() { + let body = response.text().await.unwrap_or_default(); + return Err(alien_error::AlienError::new(ErrorData::HttpResponseError { + message: format!("Azure ADC {operation} failed"), + url, + http_status: status.as_u16(), + http_request_text: None, + http_response_text: Some(body), + })); + } + + let body = response + .text() + .await + .into_alien_error() + .context(ErrorData::GenericError { + message: format!("Azure ADC {operation}: failed to read response body"), + })?; + + serde_json::from_str(&body) + .into_alien_error() + .context(ErrorData::GenericError { + message: format!("Azure ADC {operation}: unexpected response body: {body}"), + }) + } +} + +#[async_trait] +impl SandboxDataPlaneApi for AzureSandboxDataPlaneClient { + async fn create_sandbox( + &self, + group: &str, + disk: &str, + cpu: &str, + memory: &str, + ) -> Result { + let token = self.token_cache.get_bearer_token_with_scope(ADC_SCOPE).await?; + let url = self.base.build_url( + &format!("{}/sandboxes", self.group_path(group)), + Some(vec![("api-version", API_VERSION.into())]), + ); + + // `sourcesRef` is required unless a preset sandbox type is named, and resources are + // nested rather than top level. A flat {disk, cpu, memory} is rejected with + // "'sourcesRef' is required when not using a preset sandbox type". + let body = serde_json::json!({ + "sourcesRef": { "diskImage": { "name": disk, "isPublic": true } }, + "resources": { "cpu": cpu, "memory": memory }, + }) + .to_string(); + let request = AzureRequestBuilder::new(Method::PUT, url) + .content_type_json() + .content_length(&body) + .body(body) + .build()?; + + let signed = self.base.sign_request(request, &token).await?; + let response = self + .base + .execute_request(signed, "CreateSandbox", group) + .await?; + Self::parse(response, "CreateSandbox").await + } + + async fn get_sandbox(&self, group: &str, sandbox_id: &str) -> Result { + let token = self.token_cache.get_bearer_token_with_scope(ADC_SCOPE).await?; + let url = self.base.build_url( + &self.sandbox_path(group, sandbox_id), + Some(vec![("api-version", API_VERSION.into())]), + ); + + let request = AzureRequestBuilder::new(Method::GET, url).build()?; + let signed = self.base.sign_request(request, &token).await?; + let response = self + .base + .execute_request(signed, "GetSandbox", sandbox_id) + .await?; + Self::parse(response, "GetSandbox").await + } + + async fn delete_sandbox(&self, group: &str, sandbox_id: &str) -> Result<()> { + let token = self.token_cache.get_bearer_token_with_scope(ADC_SCOPE).await?; + let url = self.base.build_url( + &self.sandbox_path(group, sandbox_id), + Some(vec![("api-version", API_VERSION.into())]), + ); + + let request = AzureRequestBuilder::new(Method::DELETE, url).build()?; + let signed = self.base.sign_request(request, &token).await?; + + // Discarded on purpose: this reports that deletion started. Microsoft's own SDK says + // "poll until GET returns 404", which is what the caller does. + self.base + .execute_request(signed, "DeleteSandbox", sandbox_id) + .await?; + + Ok(()) + } + + async fn execute_shell_command( + &self, + group: &str, + sandbox_id: &str, + command: &str, + working_directory: Option, + ) -> Result { + let token = self.token_cache.get_bearer_token_with_scope(ADC_SCOPE).await?; + let url = self.base.build_url( + &format!( + "{}/executeShellCommand", + self.sandbox_path(group, sandbox_id) + ), + Some(vec![("api-version", API_VERSION.into())]), + ); + + let body = exec_body(command, working_directory).to_string(); + let request = AzureRequestBuilder::new(Method::POST, url) + .content_type_json() + .content_length(&body) + .body(body) + .build()?; + + let signed = self.base.sign_request(request, &token).await?; + let response = self + .base + .execute_request(signed, "ExecuteShellCommand", sandbox_id) + .await?; + Self::parse(response, "ExecuteShellCommand").await + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Pinned because the contract came from a preview SDK Microsoft says may change. If these + /// drift, the client must be re-read against the package rather than patched by guess. + #[test] + fn the_pinned_wire_contract_matches_what_the_sdk_ships() { + assert_eq!(API_VERSION, "2026-02-01-preview"); + assert_eq!(ADC_SCOPE, "https://management.azuredevcompute.io/.default"); + } + + /// The data-plane path has no `providers/Microsoft.App` segment; borrowing ARM's shape here + /// yields a 404 that reads like a permissions error. + #[test] + fn the_group_path_is_not_an_arm_path() { + let path = "/subscriptions/s/resourceGroups/rg/sandboxGroups/sbg1"; + assert!(!path.contains("providers")); + assert!(path.ends_with("/sandboxGroups/sbg1")); + } + + /// `workingDirectory` is the only other field the SDK sends, and dropping it would run every + /// command from the sandbox's default directory while the declaration said otherwise — + /// silently, since the data plane accepts the body either way. + #[test] + fn a_working_directory_is_sent_when_one_is_asked_for() { + let with = exec_body("ls", Some("/work".to_string())); + assert_eq!(with["workingDirectory"], "/work"); + assert_eq!(with["command"], "ls"); + + let without = exec_body("ls", None); + assert!( + without.get("workingDirectory").is_none(), + "an unasked-for directory stays absent rather than becoming an empty string" + ); + } + + #[test] + fn an_exec_result_deserializes_with_its_streams_and_code() { + let result: ExecResult = + serde_json::from_str(r#"{"stdout":"hello\n","stderr":"","exitCode":0}"#) + .expect("deserializes"); + + assert_eq!(result.stdout, "hello\n"); + assert_eq!(result.exit_code, Some(0)); + } + + /// Live responses carried only stdout and stderr. A response without an exit code must parse + /// rather than fail, and the absence must stay visible instead of defaulting to 0 — + /// a defaulted 0 would report a failed command as successful. + #[test] + fn a_missing_exit_code_is_none_rather_than_zero() { + let result: ExecResult = + serde_json::from_str(r#"{"stdout":"out","stderr":"err"}"#).expect("deserializes"); + + assert_eq!(result.exit_code, None); + } +} diff --git a/crates/alien-azure-clients/src/azure/sandbox_groups.rs b/crates/alien-azure-clients/src/azure/sandbox_groups.rs new file mode 100644 index 000000000..3f5f6694b --- /dev/null +++ b/crates/alien-azure-clients/src/azure/sandbox_groups.rs @@ -0,0 +1,250 @@ +//! Azure Container Apps SandboxGroups (`Microsoft.App/sandboxGroups`). +//! +//! The ARM control plane only. Sandboxes themselves live on a **separate** ADC data plane at +//! `management..azuredevcompute.io`, gated by the `Container Apps SandboxGroup Data +//! Owner` role — subscription Owner returns 403 against it, measured. Management +//! actions alone provision cleanly and then fail at first exec, which is why the role assignment +//! is emitted alongside the group rather than left to the operator. + +use crate::azure::common::{AzureClientBase, AzureRequestBuilder}; +use crate::azure::token_cache::AzureTokenCache; +use alien_client_core::{ErrorData, Result}; +use alien_error::{Context, IntoAlienError}; +use async_trait::async_trait; +use reqwest::Method; +use serde::{Deserialize, Serialize}; + +#[cfg(feature = "test-utils")] +use mockall::automock; + +/// ARM API version for sandbox groups. +const API_VERSION: &str = "2025-02-02-preview"; + +/// Scope every ARM call is signed for. +const ARM_SCOPE: &str = "https://management.azure.com/.default"; + +/// A sandbox group: the top-level boundary every sandbox, image, snapshot and secret sits under. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SandboxGroup { + /// ARM resource id + #[serde(skip_serializing_if = "Option::is_none")] + pub id: Option, + /// Group name + #[serde(skip_serializing_if = "Option::is_none")] + pub name: Option, + /// Region the group lives in + pub location: String, + /// Provisioning state; deletion is async, so this is not a substitute for polling to 404 + #[serde(skip_serializing_if = "Option::is_none")] + pub properties: Option, +} + +/// Observed state of a sandbox group. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SandboxGroupProperties { + /// `Succeeded`, `Failed`, `Creating`, `Deleting` + #[serde(skip_serializing_if = "Option::is_none")] + pub provisioning_state: Option, +} + +#[cfg_attr(feature = "test-utils", automock)] +#[async_trait] +pub trait SandboxGroupsApi: Send + Sync + std::fmt::Debug { + /// Creates or updates a sandbox group. + async fn create_or_update_sandbox_group( + &self, + resource_group: &str, + name: &str, + location: &str, + ) -> Result; + + /// Reads a sandbox group. A 404 surfaces as an error, which is how deletion is confirmed. + async fn get_sandbox_group(&self, resource_group: &str, name: &str) -> Result; + + /// Issues a delete. Returns before the group is gone — Azure deletes asynchronously, so the + /// caller confirms by polling `get_sandbox_group` to 404 rather than trusting this. + async fn delete_sandbox_group(&self, resource_group: &str, name: &str) -> Result<()>; +} + +/// Client for `Microsoft.App/sandboxGroups`. +#[derive(Debug)] +pub struct AzureSandboxGroupsClient { + pub base: AzureClientBase, + pub token_cache: AzureTokenCache, +} + +impl AzureSandboxGroupsClient { + /// Builds a client against the ARM management endpoint. + pub fn new(client: reqwest::Client, token_cache: AzureTokenCache) -> Self { + let endpoint = token_cache.management_endpoint().to_string(); + + Self { + base: AzureClientBase::with_client_config( + client, + endpoint, + token_cache.config().clone(), + ), + token_cache, + } + } + + fn group_path(&self, resource_group: &str, name: &str) -> String { + format!( + "/subscriptions/{}/resourceGroups/{resource_group}/providers/Microsoft.App/sandboxGroups/{name}", + self.token_cache.config().subscription_id + ) + } +} + + +impl AzureSandboxGroupsClient { + /// Reads a response body and parses it, naming the operation so a parse failure says which + /// call produced the body rather than only that some JSON was wrong. + async fn parse( + response: reqwest::Response, + operation: &str, + name: &str, + ) -> Result { + let body = response + .text() + .await + .into_alien_error() + .context(ErrorData::GenericError { + message: format!("Azure {operation}: failed to read response body for {name}"), + })?; + + serde_json::from_str(&body) + .into_alien_error() + .context(ErrorData::GenericError { + message: format!("Azure {operation}: unexpected response body for {name}: {body}"), + }) + } +} + +#[async_trait] +impl SandboxGroupsApi for AzureSandboxGroupsClient { + async fn create_or_update_sandbox_group( + &self, + resource_group: &str, + name: &str, + location: &str, + ) -> Result { + let bearer_token = self + .token_cache + .get_bearer_token_with_scope(ARM_SCOPE) + .await?; + + let url = self.base.build_url( + &self.group_path(resource_group, name), + Some(vec![("api-version", API_VERSION.into())]), + ); + + let group = SandboxGroup { + id: None, + name: None, + location: location.to_string(), + properties: None, + }; + + let body = serde_json::to_string(&group) + .into_alien_error() + .context(ErrorData::SerializationError { + message: format!("Failed to serialize sandbox group '{name}'"), + })?; + + let request = AzureRequestBuilder::new(Method::PUT, url) + .content_type_json() + .content_length(&body) + .body(body) + .build()?; + + let signed = self.base.sign_request(request, &bearer_token).await?; + let response = self + .base + .execute_request(signed, "CreateOrUpdateSandboxGroup", name) + .await?; + Self::parse(response, "CreateOrUpdateSandboxGroup", name).await + } + + async fn get_sandbox_group(&self, resource_group: &str, name: &str) -> Result { + let bearer_token = self + .token_cache + .get_bearer_token_with_scope(ARM_SCOPE) + .await?; + + let url = self.base.build_url( + &self.group_path(resource_group, name), + Some(vec![("api-version", API_VERSION.into())]), + ); + + let request = AzureRequestBuilder::new(Method::GET, url).build()?; + let signed = self.base.sign_request(request, &bearer_token).await?; + let response = self + .base + .execute_request(signed, "GetSandboxGroup", name) + .await?; + Self::parse(response, "GetSandboxGroup", name).await + } + + async fn delete_sandbox_group(&self, resource_group: &str, name: &str) -> Result<()> { + let bearer_token = self + .token_cache + .get_bearer_token_with_scope(ARM_SCOPE) + .await?; + + let url = self.base.build_url( + &self.group_path(resource_group, name), + Some(vec![("api-version", API_VERSION.into())]), + ); + + let request = AzureRequestBuilder::new(Method::DELETE, url).build()?; + let signed = self.base.sign_request(request, &bearer_token).await?; + // The response is discarded on purpose: it reports that the delete *started*. Deletion + // is confirmed by polling get_sandbox_group to 404, never by this call. + self.base + .execute_request(signed, "DeleteSandboxGroup", name) + .await?; + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_group_deserializes_from_an_arm_response() { + let group: SandboxGroup = serde_json::from_str( + r#"{"id":"/subscriptions/s/resourceGroups/rg/providers/Microsoft.App/sandboxGroups/sbg1", + "name":"sbg1","location":"swedencentral", + "properties":{"provisioningState":"Succeeded"}}"#, + ) + .expect("deserializes"); + + assert_eq!(group.name.as_deref(), Some("sbg1")); + assert_eq!( + group.properties.and_then(|p| p.provisioning_state).as_deref(), + Some("Succeeded") + ); + } + + /// A create body carries only the location; ARM rejects a read-only id or name on write. + #[test] + fn a_create_body_omits_read_only_fields() { + let group = SandboxGroup { + id: None, + name: None, + location: "swedencentral".to_string(), + properties: None, + }; + + let body = serde_json::to_value(&group).expect("serializes"); + assert_eq!(body["location"], "swedencentral"); + assert!(body.get("id").is_none()); + assert!(body.get("name").is_none()); + assert!(body.get("properties").is_none()); + } +} diff --git a/crates/alien-bindings/Cargo.toml b/crates/alien-bindings/Cargo.toml index 8c2fde321..13da1cdcf 100644 --- a/crates/alien-bindings/Cargo.toml +++ b/crates/alien-bindings/Cargo.toml @@ -11,10 +11,10 @@ crate-type = ["rlib"] [features] default = ["all-platforms"] all-platforms = ["aws", "gcp", "azure", "kubernetes", "local", "test"] # Convenience feature for all providers -aws = ["object_store/aws", "dep:base64", "dep:alien-aws-clients", "alien-client-config/aws"] -gcp = ["object_store/gcp", "dep:base64", "dep:alien-gcp-clients", "alien-client-config/gcp"] +aws = ["alien-core/sandbox-process", "object_store/aws", "dep:base64", "dep:alien-aws-clients", "alien-client-config/aws"] +gcp = ["alien-core/sandbox-process", "tokio/process", "object_store/gcp", "dep:base64", "dep:alien-gcp-clients", "alien-client-config/gcp"] azure = ["object_store/azure", "dep:base64", "dep:alien-azure-clients", "alien-client-config/azure"] -kubernetes = ["dep:alien-k8s-clients", "dep:k8s-openapi", "alien-client-config/kubernetes"] +kubernetes = ["alien-core/sandbox-process", "dep:alien-k8s-clients", "dep:k8s-openapi", "dep:base64", "alien-client-config/kubernetes"] local = ["object_store/fs", "dep:base64", "dep:oci-client", "dep:turso", "dep:sha2", "tokio/fs", "tokio/process", "alien-core/local"] test = [] # Test platform support - fast mock implementations without real cloud APIs openapi = ["dep:utoipa"] @@ -38,7 +38,7 @@ url = { workspace = true } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } chrono = { workspace = true, features = ["serde"] } -reqwest = { workspace = true, features = ["rustls-tls-webpki-roots"] } +reqwest = { workspace = true, features = ["stream", "rustls-tls-webpki-roots"] } alien-platform-api = { workspace = true, optional = true } alien-manager-api = { workspace = true, optional = true } tracing = { workspace = true } @@ -68,7 +68,7 @@ alien-gcp-clients = { workspace = true, features = ["test-utils"] } alien-azure-clients = { workspace = true, features = ["test-utils"] } axum = { workspace = true, features = ["tokio", "http1", "json"] } dotenvy = { workspace = true } -tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } +tokio = { workspace = true, features = ["macros", "rt-multi-thread", "test-util"] } tempfile = { workspace = true } temp-env = { workspace = true, features = ["async_closure"] } workspace_root = { workspace = true } diff --git a/crates/alien-bindings/src/bindings.rs b/crates/alien-bindings/src/bindings.rs index 029089eb0..496219a5e 100644 --- a/crates/alien-bindings/src/bindings.rs +++ b/crates/alien-bindings/src/bindings.rs @@ -2,15 +2,15 @@ //! //! [`Bindings`] wraps a [`crate::provider::LazyEnvBindingsProvider`], giving application //! code a small, stable surface — `storage`, `kv`, `queue`, `vault`, `container`, -//! `postgres` — instead of the full [`crate::traits::BindingsProviderApi`] used internally +//! `postgres`, `sandbox` — instead of the full [`crate::traits::BindingsProviderApi`] used internally //! by the manager and controllers. use crate::error::Result; use crate::provider::{BindingsProvider, LazyEnvBindingsProvider}; use crate::refreshing::{RefreshingKv, RefreshingQueue, RefreshingStorage, RefreshingVault}; use crate::traits::{ - BindingsProviderApi, Container, Kv, MessagePayload, Postgres, Queue, QueueMessage, Storage, - Vault, + BindingsProviderApi, Container, Kv, MessagePayload, Postgres, Queue, QueueMessage, Sandbox, + Storage, Vault, }; use std::collections::HashMap; use std::sync::Arc; @@ -186,6 +186,14 @@ impl Bindings { pub async fn postgres(&self, binding_name: &str) -> Result> { self.provider.load_postgres(binding_name).await } + + /// Loads a linked sandbox for running untrusted code. + /// + /// No refreshing wrapper: a sandbox handle addresses a control plane rather than holding + /// data-plane credentials, and each session capability is minted per call. + pub async fn sandbox(&self, binding_name: &str) -> Result> { + self.provider.load_sandbox(binding_name).await + } } #[cfg(test)] diff --git a/crates/alien-bindings/src/provider.rs b/crates/alien-bindings/src/provider.rs index 8036f6a30..18f447053 100644 --- a/crates/alien-bindings/src/provider.rs +++ b/crates/alien-bindings/src/provider.rs @@ -494,6 +494,11 @@ impl BindingsProviderApi for LazyEnvBindingsProvider { .load_service_account(binding_name) .await } + + async fn load_sandbox(&self, binding_name: &str) -> Result> { + self.ensure_binding_present(binding_name)?; + self.provider().await?.load_sandbox(binding_name).await + } } #[async_trait] @@ -1684,6 +1689,213 @@ impl BindingsProviderApi for BindingsProvider { } } } + + async fn load_sandbox(&self, binding_name: &str) -> Result> { + use alien_core::bindings::SandboxBinding; + + // Parsed before dispatch so a malformed binding fails as a config error rather than as + // a missing backend, which would send a reader looking in the wrong place. + let binding: SandboxBinding = self.parse_binding(binding_name, "sandbox")?; + + match binding { + #[cfg(feature = "local")] + SandboxBinding::Local(local_binding) => { + use crate::providers::sandbox::local::LocalSandbox; + + let sandbox: Arc = + Arc::new(LocalSandbox::new(binding_name, &local_binding).await?); + Ok(sandbox) + } + #[cfg(feature = "kubernetes")] + SandboxBinding::Kubernetes(kubernetes_binding) => { + use crate::providers::sandbox::kubernetes::KubernetesSandbox; + + let sandbox: Arc = Arc::new(KubernetesSandbox::new( + binding_name, + &kubernetes_binding, + binding_name, + )?); + Ok(sandbox) + } + #[cfg(feature = "gcp")] + SandboxBinding::Gcp(gcp_binding) => { + use crate::providers::sandbox::gcp::GcpSandbox; + + let sandbox: Arc = + Arc::new(GcpSandbox::new(binding_name, &gcp_binding)?); + Ok(sandbox) + } + #[cfg(feature = "azure")] + SandboxBinding::Azure(azure_binding) => { + use crate::providers::sandbox::azure::AzureSandbox; + use alien_azure_clients::azure::sandbox_data_plane::AzureSandboxDataPlaneClient; + use alien_azure_clients::azure::token_cache::AzureTokenCache; + + let azure_config = self.client_config.azure_config().ok_or_else(|| { + AlienError::new(ErrorData::ClientConfigInvalid { + platform: Platform::Azure, + message: "Azure config not available".to_string(), + }) + })?; + + let invalid = |field: &str| { + AlienError::new(ErrorData::BindingConfigInvalid { + binding_name: binding_name.to_string(), + env_var: alien_core::bindings::binding_env_var_name(binding_name), + reason: format!("sandbox binding field '{field}' is not a concrete value"), + }) + }; + + let group = azure_binding + .sandbox_group + .into_value(binding_name, "sandboxGroup") + .map_err(|_| invalid("sandboxGroup"))?; + let region = azure_binding + .region + .into_value(binding_name, "region") + .map_err(|_| invalid("region"))?; + let resource_group = azure_binding + .resource_group + .into_value(binding_name, "resourceGroup") + .map_err(|_| invalid("resourceGroup"))?; + + let client = AzureSandboxDataPlaneClient::new( + reqwest::Client::new(), + ®ion, + &resource_group, + AzureTokenCache::new(azure_config.clone()), + ); + + // Session ceilings come from the resource, not the caller — an application must + // not be able to raise its own by asking. + let sandbox: Arc = Arc::new(AzureSandbox::new( + Arc::new(client), + group, + "ubuntu".to_string(), + "1000m".to_string(), + "2048Mi".to_string(), + )); + Ok(sandbox) + } + #[cfg(feature = "aws")] + SandboxBinding::Aws(aws_binding) => { + use crate::providers::sandbox::aws::AwsSandbox; + use alien_aws_clients::aws::lambda_microvms::LambdaMicrovmsClient; + + let aws_config = self.client_config.aws_config().ok_or_else(|| { + AlienError::new(ErrorData::ClientConfigInvalid { + platform: Platform::Aws, + message: "AWS config not available".to_string(), + }) + })?; + + let invalid = |field: &str| { + AlienError::new(ErrorData::BindingConfigInvalid { + binding_name: binding_name.to_string(), + env_var: alien_core::bindings::binding_env_var_name(binding_name), + reason: format!("sandbox binding field '{field}' is not a concrete value"), + }) + }; + + let image_arn = aws_binding + .image_arn + .into_value(binding_name, "imageArn") + .map_err(|_| invalid("imageArn"))?; + let image_version = aws_binding + .image_version + .into_value(binding_name, "imageVersion") + .map_err(|_| invalid("imageVersion"))?; + let region = aws_binding + .region + .into_value(binding_name, "region") + .map_err(|_| invalid("region"))?; + let execution_role_arn = aws_binding + .execution_role_arn + .map(|value| { + value + .into_value(binding_name, "executionRoleArn") + .map_err(|_| invalid("executionRoleArn")) + }) + .transpose()?; + + // The sandbox lives where its image was built, which is not necessarily where + // the workload runs; signing against the workload's region would 404. + let mut config = aws_config.clone(); + config.region = region; + + let credentials = alien_aws_clients::AwsCredentialProvider::from_config(config) + .await + .context(ErrorData::BindingSetupFailed { + binding_type: "AWS sandbox".to_string(), + reason: "Failed to create credential provider".to_string(), + })?; + + let client = LambdaMicrovmsClient::new(reqwest::Client::new(), credentials); + + let egress_connector_arns = aws_binding + .egress_connector_arns + .into_iter() + .map(|value| { + value + .into_value(binding_name, "egressConnectorArns") + .map_err(|_| invalid("egressConnectorArns")) + }) + .collect::>>()?; + if egress_connector_arns.is_empty() { + // A MicroVM started with no connector reaches the public internet. Setup + // always emits one, so an empty list is a binding that did not come from a + // generated module — and starting sessions from it would quietly undo the + // declared egress policy. + return Err(AlienError::new(ErrorData::BindingConfigInvalid { + binding_name: binding_name.to_string(), + env_var: alien_core::bindings::binding_env_var_name(binding_name), + reason: "sandbox binding field 'egressConnectorArns' is empty; a MicroVM \ + started with no egress connector reaches the public internet" + .to_string(), + })); + } + + let sandbox: Arc = Arc::new(AwsSandbox::new( + Arc::new(client), + image_arn, + image_version, + execution_role_arn, + egress_connector_arns, + aws_binding.preview_ports, + aws_binding.idle_suspend_seconds, + aws_binding.max_lifetime_seconds, + )); + Ok(sandbox) + } + // A backend whose feature is off reports which one, rather than a bare + // "unsupported" that sends a reader looking at the platform instead of the build. + #[cfg(not(feature = "aws"))] + SandboxBinding::Aws(_) => Err(not_built("aws")), + #[cfg(not(feature = "azure"))] + SandboxBinding::Azure(_) => Err(not_built("azure")), + #[cfg(not(feature = "gcp"))] + SandboxBinding::Gcp(_) => Err(not_built("gcp")), + #[cfg(not(feature = "kubernetes"))] + SandboxBinding::Kubernetes(_) => Err(not_built("kubernetes")), + #[cfg(not(feature = "local"))] + SandboxBinding::Local(_) => Err(not_built("local")), + } + } + +} + +/// A binding whose backend was compiled out. +/// +/// Unused when every backend feature is on, which is the build that ships. +#[allow(dead_code)] +/// +/// Names the backend rather than reporting a bare "unsupported", which would send a reader +/// looking at the platform instead of at the build. +fn not_built(backend: &str) -> AlienError { + AlienError::new(ErrorData::OperationNotSupported { + operation: format!("load_sandbox({backend})"), + reason: "this build does not include the sandbox backend for that platform".to_string(), + }) } #[cfg(test)] diff --git a/crates/alien-bindings/src/providers/mod.rs b/crates/alien-bindings/src/providers/mod.rs index 5cda99c5d..93410405d 100644 --- a/crates/alien-bindings/src/providers/mod.rs +++ b/crates/alien-bindings/src/providers/mod.rs @@ -7,6 +7,7 @@ pub mod kv; pub(crate) mod local_store; pub mod postgres; pub mod queue; +pub mod sandbox; pub mod service_account; pub mod storage; diff --git a/crates/alien-bindings/src/providers/sandbox/agent_protocol.rs b/crates/alien-bindings/src/providers/sandbox/agent_protocol.rs new file mode 100644 index 000000000..a95d1d3a7 --- /dev/null +++ b/crates/alien-bindings/src/providers/sandbox/agent_protocol.rs @@ -0,0 +1,498 @@ +//! Speaking the sandbox agent protocol, once, for every backend that ships an agent. +//! +//! AWS and Kubernetes both talk to the same agent over HTTP and differ in exactly one thing: +//! how a request is authorized. AWS mints an endpoint token scoped to one MicroVM and an +//! explicit port set; Kubernetes claims a pod and presents a capability scoped to that session. So +//! the transport is the trait and the protocol is written once over it. +//! +//! The decoding is the reason this is shared rather than copied. A body that ends without a +//! terminal frame is a **transport failure**, not a command that finished, and a stream that +//! quietly stopped would report a truncated response as a successful command. + +use std::collections::BTreeMap; +use std::time::Duration; + +use async_trait::async_trait; +use base64::engine::general_purpose::STANDARD as BASE64; +use base64::Engine as _; +use futures::stream::BoxStream; +use futures::StreamExt; +use serde::Deserialize; +use serde_json::json; + +use crate::error::{ErrorData, Result}; +use crate::traits::{CommandOutput, RunCommandRequest}; +use alien_error::{AlienError, Context, IntoAlienError}; + +pub use alien_core::sandbox_process::AGENT_PORT; + +/// Named once: `send` treats it as the one operation a 5xx must not be retried for. +const RUN_COMMAND: &str = "sandbox.runCommand"; + +/// How a backend turns a session id into an authorized request. +/// +/// The only thing AWS and Kubernetes disagree on. +#[async_trait] +pub trait AgentTransport: Send + Sync + std::fmt::Debug { + /// Builds a request to `path` on the session's agent, carrying whatever authorizes it. + async fn request( + &self, + session_id: &str, + method: reqwest::Method, + path: &str, + ) -> Result; + + /// Name used in errors, so a failure says which backend refused. + fn provider(&self) -> &'static str; +} + +/// A frame as the agent writes it. +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase", tag = "t")] +enum AgentFrame { + Stdout { + seq: u64, + data: String, + }, + Stderr { + seq: u64, + data: String, + }, + Exit { + code: i32, + #[serde(default)] + truncated: bool, + }, + Error { + code: String, + message: String, + }, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct ReadFileResponse { + contents_base64: String, +} + +/// Runs a command, streaming frames as the agent produces them. +pub async fn run_command( + transport: &T, + session_id: &str, + request: RunCommandRequest, +) -> Result>> { + // Checked after conversion, not on the Duration: a sub-millisecond deadline is non-zero here + // and floors to `deadlineMs: 0`, which the agent then refuses as invalid. + if deadline_millis(request.deadline) == 0 { + return Err(AlienError::new(ErrorData::SandboxCommandFailed { + failure: "invalidRequest".to_string(), + reason: "a command must carry a non-zero deadline".to_string(), + })); + } + + let body = json!({ + "command": request.command, + "deadlineMs": deadline_millis(request.deadline), + "workingDirectory": request.working_directory, + "env": request.env, + }); + + let response = send( + transport + .request(session_id, reqwest::Method::POST, "/v1/exec") + .await? + .json(&body), + RUN_COMMAND, + ) + .await?; + + Ok(frame_stream(response, transport.provider())) +} + +/// Reads a file out of the sandbox. +pub async fn read_file( + transport: &T, + session_id: &str, + path: &str, +) -> Result> { + let response = send( + transport + .request(session_id, reqwest::Method::GET, "/v1/files") + .await? + .query(&[("path", path)]), + "sandbox.readFile", + ) + .await?; + + let body: ReadFileResponse = + response + .json() + .await + .into_alien_error() + .context(ErrorData::UnexpectedResponseFormat { + provider: transport.provider().to_string(), + binding_name: "sandbox.readFile".to_string(), + field: "body".to_string(), + response_json: "the agent returned a body this provider cannot parse".to_string(), + })?; + + decode( + &body.contents_base64, + transport.provider(), + "sandbox.readFile", + "contentsBase64", + ) +} + +/// Writes files into the sandbox, one request per path. +pub async fn write_files( + transport: &T, + session_id: &str, + files: BTreeMap>, +) -> Result<()> { + for (path, contents) in files { + send( + transport + .request(session_id, reqwest::Method::PUT, "/v1/files") + .await? + .json(&json!({ + "path": path, + "contentsBase64": BASE64.encode(contents), + })), + "sandbox.writeFiles", + ) + .await?; + } + + Ok(()) +} + +/// Creates a directory inside the sandbox. +pub async fn mkdir( + transport: &T, + session_id: &str, + path: &str, +) -> Result<()> { + send( + transport + .request(session_id, reqwest::Method::POST, "/v1/mkdir") + .await? + .json(&json!({ "path": path })), + "sandbox.mkdir", + ) + .await?; + + Ok(()) +} + +/// Milliseconds, saturated rather than wrapped. +/// +/// A deadline long enough to overflow `u64` milliseconds is not a deadline anyone meant, and +/// wrapping it would turn "effectively forever" into "immediately". +fn deadline_millis(deadline: Duration) -> u64 { + u64::try_from(deadline.as_millis()).unwrap_or(u64::MAX) +} + +/// Sends a request and turns a non-success into a typed error carrying the agent's own reason. +/// +/// A transport failure here is marked retryable, which holds for the file operations but not for +/// `run_command` — that request may have already started the command. Nothing retries on this +/// path today; whoever adds a retry layer has to treat `run_command` as the exception. +pub async fn send(request: reqwest::RequestBuilder, operation: &str) -> Result { + let response = request + .send() + .await + .into_alien_error() + .context(ErrorData::SandboxUnreachable { + operation: operation.to_string(), + reason: "the request never reached the agent".to_string(), + })?; + + if response.status().is_success() { + return Ok(response); + } + + let status = response.status(); + // Read the body first: the agent puts the actual cause there, and a bare status turns a + // specific refusal into a guess. + let body = response.text().await.unwrap_or_default(); + + // A 5xx is the agent failing to complete a request it accepted, which is worth another + // attempt — except for `run_command`, where the command may already be running and a retry + // would run it twice. A 4xx is a refusal: repeating it repeats the refusal. + if status.is_server_error() && operation != RUN_COMMAND { + return Err(AlienError::new(ErrorData::SandboxUnreachable { + operation: operation.to_string(), + reason: format!("the agent returned {status}: {body}"), + })); + } + + Err(AlienError::new(ErrorData::SandboxCommandFailed { + // The cause, not the operation: `reason` already names the operation, and a caller + // branching on `failure` gets an agent error code from every other construction site. + failure: "agentRefused".to_string(), + reason: format!("{operation} returned {status}: {body}"), + })) +} + +/// Turns the agent's NDJSON body into output frames. +fn frame_stream( + response: reqwest::Response, + provider: &'static str, +) -> BoxStream<'static, Result> { + struct State { + bytes: BoxStream<'static, reqwest::Result>, + buffer: Vec, + finished: bool, + saw_terminal: bool, + provider: &'static str, + } + + let state = State { + bytes: response.bytes_stream().boxed(), + buffer: Vec::new(), + finished: false, + saw_terminal: false, + provider, + }; + + futures::stream::unfold(state, |mut state| async move { + loop { + if let Some(index) = state.buffer.iter().position(|byte| *byte == b'\n') { + let line: Vec = state.buffer.drain(..=index).collect(); + let line = &line[..line.len() - 1]; + if line.is_empty() { + continue; + } + + let frame = match serde_json::from_slice::(line) { + Ok(frame) => frame, + Err(error) => { + state.finished = true; + let failure = malformed(&error.to_string(), state.provider); + return Some((Err(failure), state)); + } + }; + + if matches!(frame, AgentFrame::Exit { .. } | AgentFrame::Error { .. }) { + state.saw_terminal = true; + } + + let output = frame.into_output(state.provider); + return Some((output, state)); + } + + if state.finished { + return None; + } + + match state.bytes.next().await { + Some(Ok(chunk)) => state.buffer.extend_from_slice(&chunk), + Some(Err(error)) => { + state.finished = true; + return Some(( + Err(AlienError::new(ErrorData::SandboxUnreachable { + operation: RUN_COMMAND.to_string(), + reason: format!("the output stream failed: {error}"), + })), + state, + )); + } + None => { + state.finished = true; + if !state.saw_terminal { + return Some(( + Err(AlienError::new(ErrorData::SandboxUnreachable { + operation: RUN_COMMAND.to_string(), + reason: "the output stream ended without a terminal frame" + .to_string(), + })), + state, + )); + } + return None; + } + } + } + }) + .boxed() +} + +impl AgentFrame { + fn into_output(self, provider: &'static str) -> Result { + match self { + Self::Stdout { seq, data } => Ok(CommandOutput::Stdout { + seq, + data: decode(&data, provider, RUN_COMMAND, "data")?, + }), + Self::Stderr { seq, data } => Ok(CommandOutput::Stderr { + seq, + data: decode(&data, provider, RUN_COMMAND, "data")?, + }), + Self::Exit { code, truncated } => Ok(CommandOutput::Exit { code, truncated }), + // An error frame is the command's outcome, so it surfaces as an error rather than + // as a stream that simply stopped. + Self::Error { code, message } => { + Err(AlienError::new(ErrorData::SandboxCommandFailed { + failure: code, + reason: message, + })) + } + } + } +} + +/// `binding_name` and `field` are the caller's, not this function's: `read_file` decodes through +/// here too, and a corrupt file read reported as a runCommand output frame sends the reader to +/// the wrong place. +fn decode( + data: &str, + provider: &'static str, + binding_name: &str, + field: &str, +) -> Result> { + BASE64 + .decode(data) + .into_alien_error() + .context(ErrorData::UnexpectedResponseFormat { + provider: provider.to_string(), + binding_name: binding_name.to_string(), + field: field.to_string(), + response_json: format!("{field} was not valid base64"), + }) +} + +fn malformed(reason: &str, provider: &'static str) -> AlienError { + AlienError::new(ErrorData::UnexpectedResponseFormat { + provider: provider.to_string(), + binding_name: RUN_COMMAND.to_string(), + field: "frame".to_string(), + response_json: format!("an output frame did not parse: {reason}"), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::traits::CommandOutput; + use axum::response::IntoResponse; + use axum::routing::post; + use axum::Router; + use std::net::SocketAddr; + + async fn serve_frames(chunks: Vec<&'static str>) -> String { + let handler = move || { + let chunks = chunks.clone(); + async move { + let stream = futures::stream::iter( + chunks + .into_iter() + .map(|chunk| Ok::<_, std::io::Error>(bytes::Bytes::from(chunk))), + ); + axum::body::Body::from_stream(stream).into_response() + } + }; + + let router = Router::new().route("/v1/exec", post(handler)); + let listener = tokio::net::TcpListener::bind::("127.0.0.1:0".parse().unwrap()) + .await + .expect("bind"); + let address = listener.local_addr().expect("address"); + tokio::spawn(async move { + axum::serve(listener, router).await.expect("serve"); + }); + + format!("http://{address}") + } + + async fn frames_from(chunks: Vec<&'static str>) -> Vec> { + let base = serve_frames(chunks).await; + let response = reqwest::Client::new() + .post(format!("{base}/v1/exec")) + .send() + .await + .expect("responds"); + + frame_stream(response, "test-sandbox").collect::>().await + } + + #[tokio::test] + async fn frames_decode_in_order_with_a_real_exit_code() { + let outputs = frames_from(vec![ + "{\"t\":\"stdout\",\"seq\":0,\"data\":\"aGk=\"}\n", + "{\"t\":\"stderr\",\"seq\":1,\"data\":\"b29wcw==\"}\n", + "{\"t\":\"exit\",\"code\":7,\"truncated\":false}\n", + ]) + .await; + + assert_eq!(outputs.len(), 3); + assert_eq!( + outputs[0].as_ref().expect("stdout"), + &CommandOutput::Stdout { seq: 0, data: b"hi".to_vec() } + ); + assert_eq!( + outputs[1].as_ref().expect("stderr"), + &CommandOutput::Stderr { seq: 1, data: b"oops".to_vec() } + ); + assert_eq!( + outputs[2].as_ref().expect("exit"), + &CommandOutput::Exit { code: 7, truncated: false } + ); + } + + /// The protocol says a frame is never split across chunks; TCP makes no such promise. This + /// is the case a naive per-chunk parser gets wrong, and it fails as a parse error on + /// perfectly valid output. + #[tokio::test] + async fn a_frame_split_across_chunks_is_reassembled() { + let outputs = frames_from(vec![ + "{\"t\":\"stdo", + "ut\",\"seq\":0,\"data\":\"aGk=\"}\n{\"t\":\"ex", + "it\",\"code\":0,\"truncated\":false}\n", + ]) + .await; + + assert_eq!(outputs.len(), 2, "a split frame must not become two frames or an error"); + assert_eq!( + outputs[0].as_ref().expect("stdout"), + &CommandOutput::Stdout { seq: 0, data: b"hi".to_vec() } + ); + assert_eq!( + outputs[1].as_ref().expect("exit"), + &CommandOutput::Exit { code: 0, truncated: false } + ); + } + + /// A body that stops early looks exactly like a command that produced less + /// output — the difference is only visible in the missing terminal frame. + #[tokio::test] + async fn a_stream_without_a_terminal_frame_is_a_transport_failure() { + let outputs = frames_from(vec!["{\"t\":\"stdout\",\"seq\":0,\"data\":\"aGk=\"}\n"]).await; + + assert_eq!(outputs.len(), 2); + outputs[0].as_ref().expect("the stdout frame still arrives"); + let error = outputs[1] + .as_ref() + .expect_err("a truncated stream must not read as success"); + assert!( + error.to_string().contains("without a terminal frame"), + "the failure must name the cause: {error}" + ); + // Asserted on the code, not just the message: a dropped connection is retryable, and the + // message reads the same whichever variant carries it. + assert_eq!(error.code, "SANDBOX_UNREACHABLE", "got: {error}"); + assert!(error.retryable, "a dropped stream must stay retryable"); + } + + #[tokio::test] + async fn an_error_frame_surfaces_as_an_error_not_a_silent_end() { + let outputs = frames_from(vec![ + "{\"t\":\"error\",\"code\":\"deadlineExceeded\",\"message\":\"exceeded its 300ms deadline\"}\n", + ]) + .await; + + assert_eq!(outputs.len(), 1); + let error = outputs[0].as_ref().expect_err("an error frame is a failure"); + assert!(error.to_string().contains("deadlineExceeded"), "{error}"); + } +} diff --git a/crates/alien-bindings/src/providers/sandbox/aws.rs b/crates/alien-bindings/src/providers/sandbox/aws.rs new file mode 100644 index 000000000..99a3b5cf2 --- /dev/null +++ b/crates/alien-bindings/src/providers/sandbox/aws.rs @@ -0,0 +1,852 @@ +//! AWS sandbox provider: Lambda MicroVMs, reached over the agent protocol. +//! +//! AWS gives a transport and nothing on the other end — a MicroVM is reachable only through its +//! HTTPS endpoint, and the agent Alien ships in the image is what answers there. +//! +//! Authorization is the endpoint token, not an Alien capability. `CreateMicrovmAuthToken` is +//! minted with the workload's own IAM identity and scoped to one MicroVM, an explicit port set +//! and an expiry — a request to a port outside it is refused at the proxy. One MicroVM is one +//! session, so that scope is exactly the one a capability would express. + +use std::collections::BTreeMap; +use std::sync::Arc; + +use async_trait::async_trait; +use futures::stream::BoxStream; + +use crate::error::{ErrorData, Result}; +use crate::providers::sandbox::agent_protocol::{self, AgentTransport, AGENT_PORT}; +use crate::traits::{ + Binding, CommandOutput, CreateSessionRequest, PreviewCapability, RunCommandRequest, Sandbox, + SandboxSession, SandboxSessionState, +}; +use alien_aws_clients::aws::lambda_microvms::{ + LambdaMicrovmsApi, Microvm, MAX_AUTH_TOKEN_MINUTES, +}; +use alien_core::{Platform, SandboxCapabilities}; +use alien_error::{AlienError, Context}; + +/// Header the proxy reads to decide which port inside the MicroVM a request reaches. +const PROXY_PORT_HEADER: &str = "X-aws-proxy-port"; + +/// Life of a token minted to talk to the agent. +/// +/// Short because it is minted per request anyway: the mint response carries no expiry, so there +/// is nothing to cache against and a long-lived token would only widen the window if one leaked. +const AGENT_TOKEN_MINUTES: u32 = 5; + +/// Life of a preview capability handed to a caller. +/// +/// Clamped where it is reported, not only where it is requested: AWS caps the mint at 60 +/// minutes, so an unclamped figure here would promise a caller more life than the token has. +const PREVIEW_TOKEN_MINUTES: u32 = 30; + +/// What a caller is told a preview capability is good for. +fn preview_lifetime_seconds() -> u64 { + u64::from(PREVIEW_TOKEN_MINUTES.min(MAX_AUTH_TOKEN_MINUTES)) * 60 +} + +/// A Sandbox backed by Lambda MicroVMs. +#[derive(Debug)] +pub struct AwsSandbox { + microvms: Arc, + image_identifier: String, + image_version: String, + execution_role_arn: Option, + /// Connectors every session starts with. Empty means the public internet is reachable, so + /// `deny` is a connector rather than the absence of one. + egress_connector_arns: Vec, + /// Ports preview may be minted for. `CreateMicrovmAuthToken` grants whatever port it is + /// asked for, so this list is where "a port not listed can never be exposed" is enforced. + preview_ports: Vec, + /// Idle seconds before AWS suspends the MicroVM, where the declaration asked for it. + idle_suspend_seconds: Option, + /// Wall-clock ceiling on a session, where the declaration asked for one. Lambda terminates + /// the MicroVM when it elapses. + max_lifetime_seconds: Option, + agent: reqwest::Client, +} + +impl AwsSandbox { + /// Builds a provider over the MicroVMs API. + pub fn new( + microvms: Arc, + image_identifier: impl Into, + image_version: impl Into, + execution_role_arn: Option, + egress_connector_arns: Vec, + preview_ports: Vec, + idle_suspend_seconds: Option, + max_lifetime_seconds: Option, + ) -> Self { + Self { + microvms, + image_identifier: image_identifier.into(), + image_version: image_version.into(), + execution_role_arn, + egress_connector_arns, + preview_ports, + idle_suspend_seconds, + max_lifetime_seconds, + agent: reqwest::Client::new(), + } + } + + /// Reads a session and refuses one that is not this sandbox's own. + /// + /// The image is the boundary, and it is one per sandbox — not by naming convention but by + /// construction: the emitters bind `imageArn` to the ARN AWS assigned to the one image + /// resource emitted for this one declared sandbox, read back off that resource. Two declared + /// sandboxes cannot share an ARN however their names collide, and nothing else in the + /// codebase produces the value. Two bindings that do resolve to one image are two bindings on + /// one declared sandbox, which is the sharing the declaration asked for. + /// + /// Asked of the session itself rather than by enumerating the image. IAM does not answer it: + /// the stack binding scopes the token mint to `microvm-image:-*`, which matches + /// every sibling, so a workload holding one sandbox's handle could otherwise pass a *sibling + /// sandbox's* session id and be authorised for it. + async fn owned_microvm(&self, session_id: &str) -> Result> { + let microvm = match self.microvms.get_microvm(session_id).await { + Ok(microvm) => microvm, + // A session id that names nothing is absent, not a failure — `get` reports that as + // `None`. Read as the variant rather than as `http_status_code`: the client's own + // status-bearing variant declares no status of its own, so every error would arrive + // as 500 and this arm would never match. + Err(error) + if matches!( + &error.error, + Some(alien_client_core::ErrorData::RemoteResourceNotFound { .. }) + ) => + { + return Ok(None) + } + Err(error) => { + return Err(error).context(ErrorData::SandboxUnreachable { + operation: "sandbox.session".to_string(), + reason: format!("could not read session '{session_id}'"), + }) + } + }; + + // An absent `imageArn` is refused rather than assumed to match: it would otherwise turn a + // response the client failed to parse into a passing ownership check. + Ok(microvm + .image_arn + .as_deref() + .is_some_and(|image| image == self.image_identifier) + .then_some(microvm)) + } + + /// Refuses a session that is not one of this sandbox's own. + async fn ensure_owned(&self, session_id: &str) -> Result<()> { + if self.owned_microvm(session_id).await?.is_none() { + return Err(AlienError::new(ErrorData::SandboxUnreachable { + operation: "sandbox.session".to_string(), + reason: format!("session '{session_id}' does not belong to this sandbox"), + })); + } + Ok(()) + } + + /// Builds a request to the agent inside one session, authorised and port-scoped. + /// + /// This is the whole of what AWS does differently; everything after it is the shared agent + /// protocol. Two AWS calls per request, because the mint response carries no expiry — without + /// one, caching a token means guessing how long it stays valid. + async fn authorized_request( + &self, + session_id: &str, + method: reqwest::Method, + path: &str, + ) -> Result { + // One read serves both: the record that proves the session is ours also carries the + // endpoint to reach it. + let microvm = self.owned_microvm(session_id).await?.ok_or_else(|| { + AlienError::new(ErrorData::SandboxUnreachable { + operation: "sandbox.agent".to_string(), + reason: format!("session '{session_id}' does not belong to this sandbox"), + }) + })?; + + let endpoint = microvm.endpoint.ok_or_else(|| { + AlienError::new(ErrorData::SandboxUnreachable { + operation: "sandbox.agent".to_string(), + reason: format!("MicroVM '{session_id}' has no endpoint yet"), + }) + })?; + + let token = self + .microvms + .create_microvm_auth_token(session_id, vec![AGENT_PORT], AGENT_TOKEN_MINUTES) + .await + .context(ErrorData::SandboxUnreachable { + operation: "sandbox.agent".to_string(), + reason: format!("could not mint an endpoint token for '{session_id}'"), + })?; + + let mut request = self + .agent + .request(method, format!("https://{endpoint}{path}")) + .header(PROXY_PORT_HEADER, AGENT_PORT.to_string()); + + // The mint returns a header map, not a bearer string. Sending it as `Authorization: + // Bearer` yields a 403 that reads like a permissions problem. + for (name, value) in token.auth_token { + request = request.header(name, value); + } + + Ok(request) + } + + fn session(&self, microvm_id: String, state: Option) -> SandboxSession { + SandboxSession { + session_id: microvm_id, + state: session_state(state.as_deref()), + // Terminate destroys the MicroVM rather than fencing it, so a session never outlives + // its own generation and there is nothing for a second one to mean. + generation: 1, + } + } +} + +/// Maps a MicroVM lifecycle state onto the binding's. +fn session_state(state: Option<&str>) -> SandboxSessionState { + match state { + Some("RUNNING") => SandboxSessionState::Running, + Some("SUSPENDED") => SandboxSessionState::Suspended, + Some("TERMINATED") | Some("TERMINATING") => SandboxSessionState::Terminated, + // Anything else is a MicroVM on its way up. Reporting Running would tell a caller to + // start sending commands to something that cannot answer yet. + _ => SandboxSessionState::Starting, + } +} + +#[async_trait] +impl AgentTransport for AwsSandbox { + async fn request( + &self, + session_id: &str, + method: reqwest::Method, + path: &str, + ) -> Result { + self.authorized_request(session_id, method, path).await + } + + fn provider(&self) -> &'static str { + "aws-sandbox" + } +} + +impl Binding for AwsSandbox {} + +#[async_trait] +impl Sandbox for AwsSandbox { + fn capabilities(&self) -> SandboxCapabilities { + SandboxCapabilities::for_platform(Platform::Aws).expect("AWS has a sandbox backend") + } + + /// Starts a MicroVM. + /// + /// The client token is fresh per attempt and is **never** the caller's `session_id`. AWS + /// returns the MicroVM a token previously created even after it has been terminated, so a + /// caller reusing a session id would receive a dead MicroVM and wait for one that will never + /// start — observed against the live API. Reconnecting to an existing session is + /// [`Sandbox::get`]'s job, not an idempotency key's. + async fn create(&self, request: CreateSessionRequest) -> Result { + let _ = request.session_id; + let client_token = uuid::Uuid::new_v4().simple().to_string(); + + let microvm = self + .microvms + .run_microvm( + &self.image_identifier, + &self.image_version, + &client_token, + self.execution_role_arn.clone(), + self.egress_connector_arns.clone(), + self.idle_suspend_seconds, + self.max_lifetime_seconds, + ) + .await + .context(ErrorData::SandboxUnreachable { + operation: "sandbox.create".to_string(), + reason: format!("could not start a MicroVM from '{}'", self.image_identifier), + })?; + + let microvm_id = microvm.microvm_id.ok_or_else(|| { + AlienError::new(ErrorData::UnexpectedResponseFormat { + provider: "aws-sandbox".to_string(), + binding_name: "sandbox.create".to_string(), + field: "microvmId".to_string(), + response_json: "RunMicrovm returned no MicroVM id".to_string(), + }) + })?; + + Ok(self.session(microvm_id, microvm.state)) + } + + async fn get(&self, session_id: &str) -> Result> { + let Some(microvm) = self.owned_microvm(session_id).await? else { + return Ok(None); + }; + + // Echoing the caller's own id when the response carried none would report a session the + // client could not parse as a session it read — the same substitution `owned_microvm` + // refuses for the image. + let microvm_id = microvm.microvm_id.ok_or_else(|| { + AlienError::new(ErrorData::SandboxUnreachable { + operation: "sandbox.session".to_string(), + reason: format!("the record for session '{session_id}' carried no id"), + }) + })?; + + Ok(Some(self.session(microvm_id, microvm.state))) + } + + async fn get_or_create(&self, request: CreateSessionRequest) -> Result { + if let Some(id) = request.session_id.as_deref() { + if let Some(existing) = self.get(id).await? { + return Ok(existing); + } + } + + self.create(request).await + } + + /// Not offered, as on Azure and GCP. + /// + /// Enumerating would mean `lambda:ListMicrovms`, which AWS authorizes against no resource + /// type — the grant could only be account-wide, on the management profile any stack with a + /// sandbox holds. The + /// reason to accept that would be recovering a MicroVM whose `RunMicrovm` response never + /// arrived, since nobody holds its id. Lambda already reaps those: with no traffic to its + /// endpoint a MicroVM is suspended after the idle duration and terminated after the suspended + /// one, both 300s unless the declaration widens the first — and an orphan receives no traffic + /// by definition. A declared `maxLifetimeSeconds` bounds it outright. Reconnecting to a + /// session whose id *is* known is `get`, which reads it directly. + async fn list(&self) -> Result> { + Err(AlienError::new(ErrorData::OperationNotSupported { + operation: "sandbox.list".to_string(), + reason: "enumerating sessions would need an account-wide grant; reach a known session \ + with get, and Lambda terminates one nobody reaches" + .to_string(), + })) + } + + async fn run_command( + &self, + session_id: &str, + request: RunCommandRequest, + ) -> Result>> { + agent_protocol::run_command(self, session_id, request).await + } + + async fn read_file(&self, session_id: &str, path: &str) -> Result> { + agent_protocol::read_file(self, session_id, path).await + } + + async fn write_files(&self, session_id: &str, files: BTreeMap>) -> Result<()> { + agent_protocol::write_files(self, session_id, files).await + } + + async fn mkdir(&self, session_id: &str, path: &str) -> Result<()> { + agent_protocol::mkdir(self, session_id, path).await + } + + /// Mints a capability to reach one port inside the session. + /// + /// The endpoint is never returned bare: a caller cannot reach it without the token headers + /// and the port header, and handing over a URL would push them into building the auth + /// themselves. + async fn preview(&self, session_id: &str, port: u16) -> Result { + // Port first, ownership second: an undeclared port is refused without spending a call. + if !self.preview_ports.contains(&port) { + return Err(AlienError::new(ErrorData::OperationNotSupported { + operation: "sandbox.preview".to_string(), + reason: format!( + "port {port} is not one of this sandbox's declared preview ports {:?}; a \ + minted token would grant ingress the stack never asked for", + self.preview_ports + ), + })); + } + + // One read again: ownership and the endpoint come off the same record. + let microvm = self.owned_microvm(session_id).await?.ok_or_else(|| { + AlienError::new(ErrorData::SandboxUnreachable { + operation: "sandbox.preview".to_string(), + reason: format!("session '{session_id}' does not belong to this sandbox"), + }) + })?; + + let endpoint = microvm.endpoint.ok_or_else(|| { + AlienError::new(ErrorData::SandboxUnreachable { + operation: "sandbox.preview".to_string(), + reason: format!("MicroVM '{session_id}' has no endpoint yet"), + }) + })?; + + let token = self + .microvms + .create_microvm_auth_token(session_id, vec![port], PREVIEW_TOKEN_MINUTES) + .await + .context(ErrorData::SandboxUnreachable { + operation: "sandbox.preview".to_string(), + reason: format!("could not mint a preview token for port {port}"), + })?; + + let mut headers: BTreeMap = token.auth_token.into_iter().collect(); + headers.insert(PROXY_PORT_HEADER.to_string(), port.to_string()); + + Ok(PreviewCapability { + endpoint: format!("https://{endpoint}"), + headers, + allowed_ports: vec![port], + expires_in_seconds: preview_lifetime_seconds(), + }) + } + + async fn suspend(&self, session_id: &str) -> Result<()> { + self.ensure_owned(session_id).await?; + + self.microvms + .suspend_microvm(session_id) + .await + .context(ErrorData::SandboxUnreachable { + operation: "sandbox.suspend".to_string(), + reason: format!("could not suspend MicroVM '{session_id}'"), + }) + } + + async fn resume(&self, session_id: &str) -> Result<()> { + self.ensure_owned(session_id).await?; + + self.microvms + .resume_microvm(session_id) + .await + .context(ErrorData::SandboxUnreachable { + operation: "sandbox.resume".to_string(), + reason: format!("could not resume MicroVM '{session_id}'"), + }) + } + + async fn snapshot(&self, _session_id: &str) -> Result { + Err(AlienError::new(ErrorData::OperationNotSupported { + operation: "sandbox.snapshot".to_string(), + reason: "Lambda MicroVMs expose no snapshot API".to_string(), + })) + } + + async fn terminate(&self, session_id: &str) -> Result<()> { + self.ensure_owned(session_id).await?; + + self.microvms + .terminate_microvm(session_id) + .await + .context(ErrorData::SandboxUnreachable { + operation: "sandbox.terminate".to_string(), + reason: format!("could not terminate MicroVM '{session_id}'"), + }) + } + + fn as_any(&self) -> &dyn std::any::Any { + self + } +} + +#[cfg(test)] +mod tests { + use super::*; + use alien_aws_clients::aws::lambda_microvms::{MicrovmAuthToken, Microvm, MockLambdaMicrovmsApi}; + use std::time::Duration; + + fn image_version(version: &str) -> alien_aws_clients::aws::lambda_microvms::MicrovmImage { + alien_aws_clients::aws::lambda_microvms::MicrovmImage { + image_identifier: Some("sbx-image".to_string()), + image_arn: None, + image_version: Some(version.to_string()), + state: Some("CREATED".to_string()), + } + } + + + /// A MicroVM belonging to this sandbox's image. Ownership is now a field on the record, so + /// every fixture has to say whose session it is. + fn owned(id: &str, state: &str) -> Microvm { + Microvm { + microvm_id: Some(id.to_string()), + endpoint: None, + state: Some(state.to_string()), + image_arn: Some("sbx-image".to_string()), + image_version: Some("1".to_string()), + } + } + + fn sandbox(client: MockLambdaMicrovmsApi) -> AwsSandbox { + sandbox_previewing(client, Vec::new()) + } + + fn sandbox_previewing(client: MockLambdaMicrovmsApi, preview_ports: Vec) -> AwsSandbox { + AwsSandbox::new( + Arc::new(client), + "sbx-image", + "3", + None, + Vec::new(), + preview_ports, + None, + None, + ) + } + + /// IAM cannot draw this line: the stack binding scopes the token mint to + /// `microvm-image:-*`, which matches every sibling sandbox in the stack, so a + /// workload passing a sibling's session id would be authorised for it. The session's own + /// `imageArn` is what says whose it is. + #[tokio::test] + async fn a_session_from_another_sandbox_is_refused_before_anything_is_minted() { + let mut client = MockLambdaMicrovmsApi::new(); + client.expect_get_microvm().returning(|id| { + Ok(Microvm { + microvm_id: Some(id.to_string()), + endpoint: Some("vm.example.invalid".to_string()), + state: Some("RUNNING".to_string()), + // A live session, reachable, running — and belonging to a different sandbox. + image_arn: Some("someone-elses-image".to_string()), + image_version: Some("1".to_string()), + }) + }); + client.expect_create_microvm_auth_token().never(); + client.expect_terminate_microvm().never(); + client.expect_suspend_microvm().never(); + + let sandbox = sandbox_previewing(client, vec![8080]); + + for outcome in [ + sandbox.preview("a-siblings-session", 8080).await.err(), + sandbox.terminate("a-siblings-session").await.err(), + sandbox.suspend("a-siblings-session").await.err(), + ] { + let error = outcome.expect("a session this sandbox does not own is refused"); + assert!( + error.to_string().contains("does not belong to this sandbox"), + "names the reason: {error}" + ); + } + + assert!( + sandbox + .get("a-siblings-session") + .await + .expect("reading it is not an error") + .is_none(), + "a sibling's session reads as absent rather than as one of ours" + ); + } + + /// The absent-session path, built the way the client builds it rather than by hand. `get` + /// must report a session that does not exist as `None`, because `get_or_create` reads that + /// answer to decide whether to create one — an error there means a caller supplying a fresh + /// id can never create a session at all. + #[tokio::test] + async fn a_session_that_does_not_exist_reads_as_absent_rather_than_as_a_failure() { + let mut client = MockLambdaMicrovmsApi::new(); + client.expect_get_microvm().returning(|_| { + Err(alien_error::AlienError::new( + alien_client_core::ErrorData::RemoteResourceNotFound { + resource_type: "Microvm".to_string(), + resource_name: "GetMicrovm".to_string(), + }, + )) + }); + + assert!( + sandbox(client) + .get("never-existed") + .await + .expect("an absent session is an answer, not an error") + .is_none() + ); + } + + /// A read that genuinely failed is not an absent session. Flattening it into `None` would + /// have `get_or_create` start a second session while the first is still running. + #[tokio::test] + async fn a_failed_read_is_not_reported_as_an_absent_session() { + let mut client = MockLambdaMicrovmsApi::new(); + client.expect_get_microvm().returning(|_| { + Err(alien_error::AlienError::new( + alien_client_core::ErrorData::RateLimitExceeded { + message: "throttled".to_string(), + }, + )) + }); + + sandbox(client) + .get("ours") + .await + .expect_err("a throttle is not an absent session"); + } + + /// A response the client could not parse an image out of must not pass as ours. Defaulting + /// the other way would make every unparsed session belong to whoever asked. + #[tokio::test] + async fn a_session_with_no_image_is_not_assumed_to_be_ours() { + let mut client = MockLambdaMicrovmsApi::new(); + client.expect_get_microvm().returning(|id| { + Ok(Microvm { + microvm_id: Some(id.to_string()), + endpoint: Some("vm.example.invalid".to_string()), + state: Some("RUNNING".to_string()), + image_arn: None, + image_version: None, + }) + }); + client.expect_create_microvm_auth_token().never(); + + let error = sandbox_previewing(client, vec![8080]) + .preview("unlabelled", 8080) + .await + .expect_err("an unattributable session is refused"); + assert!(error.to_string().contains("does not belong to this sandbox")); + } + + /// The check costs one `GetMicrovm`, which `sandbox/execute` grants scoped to this image. + /// Enumerating instead would need `ListMicrovms`, which that set does not carry — an app + /// linked to a sandbox would fail on its first command. + #[tokio::test] + async fn reaching_a_session_does_not_enumerate_the_image() { + let mut client = MockLambdaMicrovmsApi::new(); + client.expect_list_microvms().never(); + client.expect_list_microvm_image_versions().never(); + client + .expect_get_microvm() + .returning(|id| Ok(owned(id, "RUNNING"))); + + let session = sandbox(client) + .get("ours") + .await + .expect("reading our own session succeeds") + .expect("it is present"); + assert_eq!(session.session_id, "ours"); + } + + /// A bare URL would be unusable: the endpoint refuses anything without the token headers and + /// the port header, so a caller handed only a string would have to rebuild the auth. + #[tokio::test] + async fn preview_returns_the_headers_a_caller_cannot_construct() { + let mut client = MockLambdaMicrovmsApi::new(); + client + .expect_list_microvm_image_versions() + .returning(|_| Ok(vec![image_version("3")])); + client.expect_list_microvms().returning(|_, _| { + Ok(vec![Microvm { microvm_id: Some("mvm-1".into()), endpoint: None, state: Some("RUNNING".into()), image_arn: Some("sbx-image".to_string()), image_version: Some("1".to_string()) }]) + }); + client.expect_get_microvm().returning(|_| { + Ok(Microvm { microvm_id: Some("mvm-1".to_string()), endpoint: Some("mvm-1.lambda-microvms.aws".to_string()), state: Some("RUNNING".to_string()), image_arn: Some("sbx-image".to_string()), image_version: Some("1".to_string()) }) + }); + client + .expect_create_microvm_auth_token() + .withf(|_, ports, minutes| ports.as_slice() == [8080] && *minutes == PREVIEW_TOKEN_MINUTES) + .returning(|_, _, _| { + Ok(MicrovmAuthToken { + auth_token: std::collections::HashMap::from([( + "X-aws-proxy-auth".to_string(), + "jwe-value".to_string(), + )]), + }) + }); + + let capability = sandbox_previewing(client, vec![8080]) + .preview("mvm-1", 8080) + .await + .expect("mints"); + + assert_eq!(capability.endpoint, "https://mvm-1.lambda-microvms.aws"); + assert_eq!(capability.headers.get("X-aws-proxy-auth").map(String::as_str), Some("jwe-value")); + assert_eq!(capability.headers.get(PROXY_PORT_HEADER).map(String::as_str), Some("8080")); + assert_eq!(capability.allowed_ports, vec![8080]); + assert_eq!(capability.expires_in_seconds, 1800); + } + + /// The declared list is where ingress is bounded: `CreateMicrovmAuthToken` mints a token for + /// whatever port it is handed, so an unlisted port must be refused before the call rather + /// than after it. + #[tokio::test] + async fn a_port_the_stack_did_not_declare_is_refused_before_a_token_exists() { + let mut client = MockLambdaMicrovmsApi::new(); + client.expect_get_microvm().never(); + client.expect_create_microvm_auth_token().never(); + + let error = sandbox_previewing(client, vec![8080]) + .preview("mvm-1", 22) + .await + .expect_err("port 22 was never declared"); + + assert!( + error.to_string().contains("22"), + "the refusal must name the port asked for: {error}" + ); + } + + /// The figure handed to a caller must not outrun the token behind it: AWS caps the mint at + /// 60 minutes, so an unclamped 30-minute promise would still be honest, but a raised + /// `PREVIEW_TOKEN_MINUTES` past the cap would not. + #[test] + fn a_reported_preview_lifetime_never_exceeds_what_aws_will_mint() { + assert_eq!(preview_lifetime_seconds(), 1800); + assert!( + preview_lifetime_seconds() <= u64::from(MAX_AUTH_TOKEN_MINUTES) * 60, + "the reported lifetime must not outrun the cap the client sends" + ); + } + + /// The lifecycle states AWS reports, mapped onto the binding's. Read through `get`, which is + /// the only way a session is reached now that enumeration is gone. + #[tokio::test] + async fn a_microvm_that_is_not_running_yet_is_reported_as_starting() { + for (aws_state, expected) in [ + ("PENDING", SandboxSessionState::Starting), + ("RUNNING", SandboxSessionState::Running), + ("SUSPENDED", SandboxSessionState::Suspended), + ("TERMINATED", SandboxSessionState::Terminated), + ] { + let mut client = MockLambdaMicrovmsApi::new(); + client + .expect_get_microvm() + .returning(move |id| Ok(owned(id, aws_state))); + + let session = sandbox(client) + .get("s1") + .await + .expect("reads") + .expect("present"); + assert_eq!(session.state, expected, "AWS state {aws_state}"); + } + } + + /// The declared ceiling has to survive the last hop as well as the first: the binding carries + /// it onto `AwsSandbox`, and only this call puts it on the wire. A field dropped here would + /// leave a sandbox running past a limit its stack declared, with every other test still green. + #[tokio::test] + async fn the_declared_lifetime_reaches_the_run_call() { + let mut client = MockLambdaMicrovmsApi::new(); + client + .expect_run_microvm() + .withf(|_, _, _, _, _, _, max_lifetime| *max_lifetime == Some(1800)) + .returning(|_, _, _, _, _, _, _| Ok(owned("mvm-1", "PENDING"))); + + AwsSandbox::new( + std::sync::Arc::new(client), + "sbx-image", + "3", + None, + vec!["connector".to_string()], + Vec::new(), + None, + Some(1800), + ) + .create(CreateSessionRequest { + session_id: None, + tenant_key: None, + env: BTreeMap::new(), + }) + .await + .expect("the run carries the declared ceiling"); + } + + /// Observed live: AWS returns the MicroVM a client token previously created **even after it + /// is terminated**. Using the caller's session id as that token hands back a dead MicroVM + /// and then waits for it to start, which is a hang, not an error. + #[tokio::test] + async fn a_caller_supplied_session_id_is_never_the_client_token() { + let mut client = MockLambdaMicrovmsApi::new(); + client + .expect_run_microvm() + .withf(|image, version, token, _, _, _, _| { + image == "sbx-image" && version == "3" && token != "caller-chosen" + }) + .returning(|_, _, _, _, _, _, _| { + Ok(Microvm { microvm_id: Some("mvm-9".to_string()), endpoint: None, state: Some("PENDING".to_string()), image_arn: Some("sbx-image".to_string()), image_version: Some("1".to_string()) }) + }); + + let session = sandbox(client) + .create(CreateSessionRequest { + session_id: Some("caller-chosen".to_string()), + tenant_key: None, + env: BTreeMap::new(), + }) + .await + .expect("creates"); + + assert_eq!(session.session_id, "mvm-9", "AWS assigns the id, so that is what is returned"); + } + + #[tokio::test] + async fn a_command_without_a_deadline_is_refused_before_any_aws_call() { + // No expectations set: a call to AWS here would fail the mock, which is the assertion. + let outcome = sandbox(MockLambdaMicrovmsApi::new()) + .run_command( + "mvm-1", + RunCommandRequest { + command: vec!["/bin/echo".to_string()], + working_directory: None, + env: BTreeMap::new(), + deadline: Duration::ZERO, + }, + ) + .await; + + match outcome { + Ok(_) => panic!("a zero deadline must be refused"), + Err(error) => assert!(error.to_string().contains("non-zero deadline"), "{error}"), + } + } + + /// A rolled image version does not end the sessions running on the previous one, and does + /// not change whose they are. Comparing the version as well as the image would make `get` + /// return None for a live session after a roll, which a caller reads as expired — the exact + /// false negative GCP's capability set refuses to ship. + #[tokio::test] + async fn a_session_on_a_previous_image_version_is_still_ours() { + let mut client = MockLambdaMicrovmsApi::new(); + client.expect_get_microvm().returning(|id| { + Ok(Microvm { + microvm_id: Some(id.to_string()), + endpoint: None, + state: Some("RUNNING".to_string()), + image_arn: Some("sbx-image".to_string()), + // The binding is pinned to version 3; this session predates the roll. + image_version: Some("2".to_string()), + }) + }); + + let found = sandbox(client) + .get("older") + .await + .expect("reads") + .expect("a session on the previous version is still live and still ours"); + + assert_eq!(found.session_id, "older"); + } + + /// Enumeration would cost an account-wide `ListMicrovms`, and the case it would serve — + /// a `RunMicrovm` whose response never arrived, leaving a MicroVM nobody holds the id for — + /// is already handled by Lambda: no traffic reaches an orphan's endpoint, so it suspends + /// after the idle duration and is terminated after the suspended one. + #[tokio::test] + async fn sessions_are_not_enumerable_and_nothing_asks_aws_to_be() { + let mut client = MockLambdaMicrovmsApi::new(); + client.expect_list_microvms().never(); + client.expect_list_microvm_image_versions().never(); + + let error = sandbox(client) + .list() + .await + .expect_err("listing is not offered on AWS"); + assert!( + error.to_string().contains("get"), + "points the caller at what does work: {error}" + ); + } +} diff --git a/crates/alien-bindings/src/providers/sandbox/azure.rs b/crates/alien-bindings/src/providers/sandbox/azure.rs new file mode 100644 index 000000000..aeedc0f24 --- /dev/null +++ b/crates/alien-bindings/src/providers/sandbox/azure.rs @@ -0,0 +1,467 @@ +//! Azure sandbox provider. +//! +//! The one backend with no Alien agent inside the sandbox: the ADC data plane implements exec, +//! files and lifecycle natively, so this provider is a translation layer rather than a transport +//! for a protocol. Verified against a stock `ubuntu` catalog disk containing no Alien code. + +use std::collections::BTreeMap; + +use async_trait::async_trait; +use futures::stream::{self, BoxStream}; + +use crate::error::{ErrorData, Result}; +use crate::traits::{ + Binding, CommandOutput, CreateSessionRequest, PreviewCapability, RunCommandRequest, Sandbox, + SandboxSession, SandboxSessionState, +}; +use alien_azure_clients::azure::sandbox_data_plane::SandboxDataPlaneApi; +use alien_client_core::ErrorData as ClientErrorData; +use alien_core::{Platform, SandboxCapabilities}; +use alien_error::AlienError; + +/// A Sandbox backed by the Azure ADC data plane. +#[derive(Debug)] +pub struct AzureSandbox { + client: std::sync::Arc, + sandbox_group: String, + /// Disk image every session is created from. + disk: String, + /// Session ceilings, in the data plane's own units. + cpu: String, + memory: String, +} + +impl AzureSandbox { + /// Builds a provider bound to one sandbox group. + pub fn new( + client: std::sync::Arc, + sandbox_group: String, + disk: String, + cpu: String, + memory: String, + ) -> Self { + Self { + client, + sandbox_group, + disk, + cpu, + memory, + } + } + + fn unsupported(&self, capability: &str) -> AlienError { + AlienError::new(ErrorData::OperationNotSupported { + operation: capability.to_string(), + reason: "not supported on azure".to_string(), + }) + } + + fn failed(operation: &str, error: impl std::fmt::Display) -> AlienError { + AlienError::new(ErrorData::OperationNotSupported { + operation: operation.to_string(), + reason: format!("the Azure sandbox data plane refused the call: {error}"), + }) + } +} + +impl Binding for AzureSandbox {} + +#[async_trait] +impl Sandbox for AzureSandbox { + fn capabilities(&self) -> SandboxCapabilities { + SandboxCapabilities::for_platform(Platform::Azure).expect("Azure has a sandbox backend") + } + + async fn create(&self, request: CreateSessionRequest) -> Result { + let sandbox = self + .client + .create_sandbox(&self.sandbox_group, &self.disk, &self.cpu, &self.memory) + .await + .map_err(|error| Self::failed("sandbox.create", error))?; + + // The caller's requested id is not authoritative: Azure allocates the id, and returning + // the requested one would hand back a handle that addresses nothing. + let _ = request.session_id; + + Ok(SandboxSession { + session_id: sandbox.id, + state: SandboxSessionState::Running, + generation: 1, + }) + } + + async fn get(&self, session_id: &str) -> Result> { + match self.client.get_sandbox(&self.sandbox_group, session_id).await { + Ok(sandbox) => Ok(Some(SandboxSession { + session_id: sandbox.id, + state: match sandbox.status.as_deref() { + Some("Stopped") => SandboxSessionState::Suspended, + _ => SandboxSessionState::Running, + }, + generation: 1, + })), + // A 404 is "gone", which is a valid answer. Anything else is a real failure and must + // not be flattened into None, or a throttle would read as an expired session. + Err(error) if is_not_found(&error) => Ok(None), + Err(error) => Err(Self::failed("sandbox.get", error)), + } + } + + async fn get_or_create(&self, request: CreateSessionRequest) -> Result { + if let Some(id) = request.session_id.as_deref() { + if let Some(existing) = self.get(id).await? { + return Ok(existing); + } + } + + self.create(request).await + } + + async fn list(&self) -> Result> { + Err(self.unsupported("list")) + } + + async fn run_command( + &self, + session_id: &str, + request: RunCommandRequest, + ) -> Result>> { + if request.deadline.is_zero() { + return Err(AlienError::new(ErrorData::OperationNotSupported { + operation: "sandbox.runCommand".to_string(), + reason: "a command must carry a non-zero deadline".to_string(), + })); + } + + // The deadline bounds the untrusted code, not the caller's patience. Read out of the + // preview SDK rather than assumed: `executeShellCommand` sends `command` and an optional + // `workingDirectory` and nothing else, so there is no server-side timeout to ask for and + // the only lever that stops an overrun is ending the session. The call returns once that + // is confirmed, which is after the deadline — reporting containment before it held would + // be the claim this whole path exists to make good on. + let result = match tokio::time::timeout( + request.deadline, + self.client + .execute_shell_command( + &self.sandbox_group, + session_id, + &request.command.join(" "), + request.working_directory.clone(), + ), + ) + .await + { + Ok(inner) => inner.map_err(|error| Self::failed("sandbox.runCommand", error))?, + Err(_) => { + self.terminate(session_id).await?; + return Err(AlienError::new(ErrorData::SandboxCommandFailed { + failure: "deadlineExceeded".to_string(), + reason: format!( + "the command exceeded its {}s deadline and the session was terminated", + request.deadline.as_secs() + ), + })); + } + }; + + // The data plane returns a completed result, not a stream, so the frames are + // reconstructed in order. Streaming is unverified on Azure, and pretending otherwise + // here would be inventing a guarantee. + let mut frames: Vec> = Vec::new(); + if !result.stdout.is_empty() { + frames.push(Ok(CommandOutput::Stdout { + seq: 0, + data: result.stdout.into_bytes(), + })); + } + if !result.stderr.is_empty() { + frames.push(Ok(CommandOutput::Stderr { + seq: frames.len() as u64, + data: result.stderr.into_bytes(), + })); + } + + frames.push(Ok(CommandOutput::Exit { + // A missing exit code is not success. Azure did not report one, so the command's + // outcome is unknown, and -1 says that rather than claiming zero. + code: result.exit_code.unwrap_or(-1), + truncated: false, + })); + + Ok(Box::pin(stream::iter(frames))) + } + + async fn read_file(&self, _session_id: &str, _path: &str) -> Result> { + Err(self.unsupported("readFile")) + } + + async fn write_files(&self, _session_id: &str, _files: BTreeMap>) -> Result<()> { + Err(self.unsupported("writeFiles")) + } + + async fn mkdir(&self, _session_id: &str, _path: &str) -> Result<()> { + Err(self.unsupported("mkdir")) + } + + async fn preview(&self, _session_id: &str, _port: u16) -> Result { + Err(self.unsupported("preview")) + } + + async fn suspend(&self, _session_id: &str) -> Result<()> { + Err(self.unsupported("suspendResume")) + } + + async fn resume(&self, _session_id: &str) -> Result<()> { + Err(self.unsupported("suspendResume")) + } + + async fn snapshot(&self, _session_id: &str) -> Result { + Err(self.unsupported("snapshot")) + } + + async fn terminate(&self, session_id: &str) -> Result<()> { + match self.client.delete_sandbox(&self.sandbox_group, session_id).await { + Ok(_) => {} + // An already-gone session is the desired end state. Every other failure leaves the + // session running, and reporting success there tells the caller untrusted code has + // stopped when it has not. + Err(error) if is_not_found(&error) => return Ok(()), + Err(error) => return Err(Self::failed("sandbox.terminate", error)), + } + + // The delete is accepted, not completed: the client's own contract is "returns before it + // is gone; confirm by polling to 404". Returning here would report containment while the + // code is still running, which is the whole point of terminate. + for _ in 0..TERMINATE_POLL_ATTEMPTS { + if self.get(session_id).await?.is_none() { + return Ok(()); + } + tokio::time::sleep(TERMINATE_POLL_INTERVAL).await; + } + + Err(AlienError::new(ErrorData::SandboxUnreachable { + operation: "sandbox.terminate".to_string(), + reason: format!( + "deletion of '{session_id}' was accepted but the session was still present after {}s; it may still be running", + TERMINATE_POLL_ATTEMPTS * TERMINATE_POLL_INTERVAL.as_secs() as u32 + ), + })) + } + + fn as_any(&self) -> &dyn std::any::Any { + self + } +} + +/// How long termination waits for Azure to actually remove a session. +/// +/// Azure accepts a delete and completes it asynchronously, so "gone" is only observable by +/// polling. Bounded rather than open-ended: a caller waiting forever is its own outage, and an +/// unconfirmed deletion is reported as unconfirmed rather than silently treated as done. +const TERMINATE_POLL_ATTEMPTS: u32 = 15; +const TERMINATE_POLL_INTERVAL: std::time::Duration = std::time::Duration::from_secs(2); + +/// Whether an Azure data-plane failure means the session is already gone. +/// +/// Reads the status the client carries rather than the rendered message: `AlienError`'s `Display` +/// walks the whole source chain and the data plane puts the response body in it, so a path or a +/// trace id containing "404" would otherwise turn a throttle into "gone". +fn is_not_found(error: &AlienError) -> bool { + // Both variants, because the client wraps: `create_azure_http_error_with_context` builds the + // `HttpResponseError` carrying the status and then returns + // `http_error.context(RemoteResourceNotFound)` for a 404, so the outer variant is the + // classified one and the status only survives on the source. + matches!( + &error.error, + Some(ClientErrorData::RemoteResourceNotFound { .. }) + ) || matches!( + &error.error, + Some(ClientErrorData::HttpResponseError { http_status, .. }) if *http_status == 404 + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use alien_azure_clients::azure::sandbox_data_plane::MockSandboxDataPlaneApi; + + fn http_error(status: u16, body: &str) -> AlienError { + AlienError::new(ClientErrorData::HttpResponseError { + message: "Azure ADC sandbox.get failed".to_string(), + url: "https://example.invalid/sandboxes/s1".to_string(), + http_status: status, + http_request_text: None, + http_response_text: Some(body.to_string()), + }) + } + + fn sandbox_with(client: MockSandboxDataPlaneApi) -> AzureSandbox { + AzureSandbox::new( + std::sync::Arc::new(client), + "grp".to_string(), + "ubuntu".to_string(), + "1000m".to_string(), + "2048Mi".to_string(), + ) + } + + /// Azure accepts a delete and completes it later, so returning on the accepted call would + /// report that untrusted code had stopped while it was still running. Time is paused, so the + /// poll runs to its bound instantly. + #[tokio::test(start_paused = true)] + async fn a_termination_that_never_completes_is_reported_as_unconfirmed() { + let mut client = MockSandboxDataPlaneApi::new(); + client.expect_delete_sandbox().returning(|_, _| Ok(())); + client.expect_get_sandbox().returning(|_, id| { + Ok(alien_azure_clients::azure::sandbox_data_plane::Sandbox { + id: id.to_string(), + status: Some("Running".to_string()), + }) + }); + + let error = sandbox_with(client) + .terminate("s1") + .await + .expect_err("a session still present after the poll is not contained"); + assert!( + error.to_string().contains("may still be running"), + "says what is not known: {error}" + ); + } + + /// The same path when Azure does finish: the session becomes absent and terminate succeeds. + #[tokio::test(start_paused = true)] + async fn a_termination_is_confirmed_once_the_session_is_gone() { + let mut client = MockSandboxDataPlaneApi::new(); + client.expect_delete_sandbox().returning(|_, _| Ok(())); + client + .expect_get_sandbox() + .returning(|_, _| Err(http_error(404, "SandboxNotFound"))); + + sandbox_with(client) + .terminate("s1") + .await + .expect("an absent session is a confirmed termination"); + } + + /// The discriminating case. A throttle whose body mentions 404 — a trace id, an inner code, a + /// path — used to read as "the session is gone", which starts a second sandbox while the + /// first keeps running and reports a live session as terminated. + #[test] + fn only_the_status_decides_whether_a_session_is_gone() { + assert!(is_not_found(&http_error(404, "SandboxNotFound"))); + + // The shape the client actually produces: a 404 is returned as + // `http_error.context(RemoteResourceNotFound)`, so the outer variant is the classified + // one. Matching only `HttpResponseError` made every real 404 read as a live session. + assert!( + is_not_found(&AlienError::new(ClientErrorData::RemoteResourceNotFound { + resource_type: "Sandbox".to_string(), + resource_name: "s1".to_string(), + })), + "a wrapped 404 is how the client reports an absent session" + ); + + assert!( + !is_not_found(&http_error(429, "throttled; see trace 404abc")), + "a throttle is not a missing session" + ); + assert!( + !is_not_found(&http_error(403, "denied on /sandboxes/404/read")), + "a path containing 404 is not a missing session" + ); + assert!( + !is_not_found(&http_error(500, "internal error 404")), + "a server failure is not a missing session" + ); + } + + /// A data plane whose exec never returns, so the only thing that can end the call is the + /// deadline. Hand-written rather than mocked because mockall resolves an async expectation + /// immediately, which is the one thing this test needs not to happen. + #[derive(Debug)] + struct HangingExec { + deleted: std::sync::Arc, + } + + #[async_trait] + impl SandboxDataPlaneApi for HangingExec { + async fn create_sandbox( + &self, + _group: &str, + _disk: &str, + _cpu: &str, + _memory: &str, + ) -> alien_client_core::Result + { + unreachable!("the deadline path never creates") + } + + async fn get_sandbox( + &self, + _group: &str, + _sandbox_id: &str, + ) -> alien_client_core::Result + { + Err(http_error(404, "SandboxNotFound")) + } + + async fn delete_sandbox(&self, _group: &str, _sandbox_id: &str) -> alien_client_core::Result<()> { + self.deleted + .store(true, std::sync::atomic::Ordering::SeqCst); + Ok(()) + } + + async fn execute_shell_command( + &self, + _group: &str, + _sandbox_id: &str, + _command: &str, + _working_directory: Option, + ) -> alien_client_core::Result + { + std::future::pending().await + } + } + + /// The deadline bounds untrusted code, not the caller's patience. The data plane takes no + /// timeout, so reporting `deadlineExceeded` while the command kept running would be the + /// containment claim this resource exists to make, unbacked. Time is paused, so the deadline + /// arrives instantly. + #[tokio::test(start_paused = true)] + async fn a_command_past_its_deadline_takes_the_session_with_it() { + let deleted = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let sandbox = AzureSandbox::new( + std::sync::Arc::new(HangingExec { + deleted: deleted.clone(), + }), + "grp".to_string(), + "ubuntu".to_string(), + "1000m".to_string(), + "2048Mi".to_string(), + ); + + let error = sandbox + .run_command( + "s1", + RunCommandRequest { + command: vec!["sleep".to_string(), "forever".to_string()], + working_directory: None, + env: BTreeMap::new(), + deadline: std::time::Duration::from_secs(30), + }, + ) + .await + .err() + .expect("a command that outran its deadline has not succeeded"); + + assert!( + error.to_string().contains("deadlineExceeded"), + "the caller has to be able to tell this apart from a command that failed: {error}" + ); + assert!( + deleted.load(std::sync::atomic::Ordering::SeqCst), + "the session must actually be deleted, not merely reported as terminated" + ); + } +} diff --git a/crates/alien-bindings/src/providers/sandbox/gcp.rs b/crates/alien-bindings/src/providers/sandbox/gcp.rs new file mode 100644 index 000000000..70e77d26f --- /dev/null +++ b/crates/alien-bindings/src/providers/sandbox/gcp.rs @@ -0,0 +1,554 @@ +//! GCP sandbox provider. +//! +//! A Cloud Run sandbox is a subprocess of the workload's own instance, created through a CLI on +//! the container's filesystem. There is no control plane to call, no credential to hold and no +//! capability to mint: the boundary is the launcher, and the launcher is already there. +//! +//! Every command is passed as argv rather than a shell string, including file paths and file +//! contents, so nothing a caller supplies is ever parsed by a shell. + +use std::collections::BTreeMap; +use std::time::Duration; + +use async_trait::async_trait; +use base64::engine::general_purpose::STANDARD as BASE64; +use base64::Engine as _; +use futures::stream::BoxStream; +use futures::StreamExt; +use tokio::sync::mpsc; + +use crate::error::{ErrorData, Result}; +use crate::traits::{ + Binding, CommandOutput, CreateSessionRequest, PreviewCapability, RunCommandRequest, Sandbox, + SandboxSession, SandboxSessionState, +}; +use alien_core::bindings::GcpSandboxBinding; +use alien_core::sandbox_process::{self, ProcessFrame, ProcessStream, FRAME_CHANNEL_DEPTH}; +use alien_core::{Platform, SandboxCapabilities}; +use alien_error::AlienError; + +/// How much of one command's output is kept before the terminal frame reports truncation. +const OUTPUT_CAP: usize = 8 * 1024 * 1024; + +/// Ceiling on a launcher call that is not the caller's command, such as a create or a delete. +const CONTROL_DEADLINE: Duration = Duration::from_secs(60); + +/// A Sandbox backed by the Cloud Run sandbox launcher. +#[derive(Debug)] +pub struct GcpSandbox { + launcher_path: String, + allow_egress: bool, + binding_name: String, +} + +impl GcpSandbox { + /// Builds a provider from its binding. + pub fn new(binding_name: &str, binding: &GcpSandboxBinding) -> Result { + let launcher_path = binding + .launcher_path + .clone() + .into_value(binding_name, "launcherPath") + .map_err(|error| { + AlienError::new(ErrorData::BindingConfigInvalid { + binding_name: binding_name.to_string(), + env_var: alien_core::bindings::binding_env_var_name(binding_name), + reason: error.to_string(), + }) + })?; + + let allow_egress = binding + .allow_egress + .clone() + .into_value(binding_name, "allowEgress") + .map_err(|error| { + AlienError::new(ErrorData::BindingConfigInvalid { + binding_name: binding_name.to_string(), + env_var: alien_core::bindings::binding_env_var_name(binding_name), + reason: error.to_string(), + }) + })?; + + Ok(Self { + launcher_path, + allow_egress, + binding_name: binding_name.to_string(), + }) + } + + /// Runs the launcher and returns its stdout, failing on a non-zero exit. + /// + /// Used for the control verbs. A caller's own command goes through [`Self::frames`] instead, + /// which streams rather than collecting. + async fn control(&self, operation: &str, arguments: &[String]) -> Result> { + let child = sandbox_process::spawn(&self.launcher_path, arguments) + .and_then(|mut command| command.spawn()) + .map_err(|error| self.failed(operation, &format!("launcher would not start: {error}")))?; + + let frames = sandbox_process::run(child, CONTROL_DEADLINE, OUTPUT_CAP).await; + + let mut stdout = Vec::new(); + let mut stderr = Vec::new(); + for frame in &frames { + match frame { + ProcessFrame::Output { + stream: ProcessStream::Stdout, + data, + .. + } => stdout.extend_from_slice(data), + ProcessFrame::Output { + stream: ProcessStream::Stderr, + data, + .. + } => stderr.extend_from_slice(data), + _ => {} + } + } + + match frames.last() { + Some(ProcessFrame::Exit { code: 0, .. }) => Ok(stdout), + // stderr, not the exit code alone: the launcher puts the actual cause there, and a + // bare status turns a specific failure into a guess. + Some(ProcessFrame::Exit { code, .. }) => Err(self.failed( + operation, + &format!( + "launcher exited with {code}: {}", + String::from_utf8_lossy(&stderr).trim() + ), + )), + Some(ProcessFrame::Failed { code, message }) => { + Err(self.failed(operation, &format!("{code}: {message}"))) + } + _ => Err(self.failed(operation, "launcher produced no terminal frame")), + } + } + + fn failed(&self, operation: &str, reason: &str) -> AlienError { + AlienError::new(ErrorData::OperationNotSupported { + operation: operation.to_string(), + reason: format!("{reason} (binding '{}')", self.binding_name), + }) + } + + fn unsupported(&self, capability: &str, reason: &str) -> AlienError { + AlienError::new(ErrorData::OperationNotSupported { + operation: capability.to_string(), + reason: reason.to_string(), + }) + } + + /// Refuses a path that traverses upward, the same lexical rule the in-sandbox agent applies. + fn checked_path(&self, path: &str, operation: &str) -> Result { + if path.is_empty() || path.split('/').any(|part| part == "..") { + return Err(self.failed(operation, &format!("path '{path}' traverses upward"))); + } + Ok(path.to_string()) + } + + /// Builds `sandbox exec -- `. + fn exec_arguments(&self, session_id: &str, command: &[String]) -> Vec { + let mut arguments = vec![ + "exec".to_string(), + session_id.to_string(), + "--".to_string(), + ]; + arguments.extend(command.iter().cloned()); + arguments + } +} + +impl Binding for GcpSandbox {} + +#[async_trait] +impl Sandbox for GcpSandbox { + fn as_any(&self) -> &dyn std::any::Any { + self + } + + fn capabilities(&self) -> SandboxCapabilities { + SandboxCapabilities::for_platform(Platform::Gcp).expect("GCP has a sandbox backend") + } + + /// Starts a sandbox with a caller-chosen id. + /// + /// Egress comes from the binding rather than the request: the launcher decides it at create + /// time and an application must not be able to widen its own. + async fn create(&self, request: CreateSessionRequest) -> Result { + let session_id = request + .session_id + .unwrap_or_else(|| uuid::Uuid::new_v4().simple().to_string()); + + let mut arguments = vec!["run".to_string(), "--id".to_string(), session_id.clone()]; + if self.allow_egress { + arguments.push("--allow-egress".to_string()); + } + + self.control("sandbox.create", &arguments).await?; + + Ok(SandboxSession { + session_id, + state: SandboxSessionState::Running, + // A sandbox is destroyed rather than fenced, so a session never outlives its own + // generation and there is nothing for a second one to mean. + generation: 1, + }) + } + + /// Reconnecting is not offered, and the reason is measured rather than assumed. + async fn get(&self, _session_id: &str) -> Result> { + Err(self.unsupported( + "reconnect", + "a Cloud Run sandbox id is scoped to one instance, and session affinity held 2 of \ + 100 five-turn conversations", + )) + } + + async fn get_or_create(&self, request: CreateSessionRequest) -> Result { + self.create(request).await + } + + async fn list(&self) -> Result> { + Err(self.unsupported( + "reconnect", + "the launcher has no enumeration verb, and an id reaches only the instance that \ + created it", + )) + } + + async fn run_command( + &self, + session_id: &str, + request: RunCommandRequest, + ) -> Result>> { + if request.command.is_empty() { + return Err(self.failed("sandbox.runCommand", "command is empty")); + } + + let mut arguments = self.exec_arguments(session_id, &request.command); + if let Some(directory) = &request.working_directory { + // Prepended rather than appended: everything after `--` is the caller's command. + arguments.insert(2, directory.clone()); + arguments.insert(2, "--workdir".to_string()); + } + + let child = sandbox_process::spawn(&self.launcher_path, &arguments) + .and_then(|mut command| command.spawn()) + .map_err(|error| { + self.failed( + "sandbox.runCommand", + &format!("launcher would not start: {error}"), + ) + })?; + + let (sender, receiver) = mpsc::channel(FRAME_CHANNEL_DEPTH); + tokio::spawn(sandbox_process::stream( + child, + request.deadline, + OUTPUT_CAP, + sender, + )); + + // A failed frame becomes a stream error rather than a fabricated exit code: a deadline + // that killed the command is not the command reporting -1. + Ok(futures::stream::unfold(receiver, |mut receiver| async move { + let frame = receiver.recv().await?; + let item = match frame { + ProcessFrame::Failed { code, message } => Err(AlienError::new( + ErrorData::OperationNotSupported { + operation: "sandbox.runCommand".to_string(), + reason: format!("{code}: {message}"), + }, + )), + other => Ok(CommandOutput::from(other)), + }; + Some((item, receiver)) + }) + .boxed()) + } + + async fn read_file(&self, session_id: &str, path: &str) -> Result> { + let path = self.checked_path(path, "sandbox.readFile")?; + let command = vec!["/bin/cat".to_string(), path]; + self.control("sandbox.readFile", &self.exec_arguments(session_id, &command)) + .await + } + + /// Writes files by handing the contents to the sandbox base64-encoded **as an argument**. + /// + /// Not interpolated into a shell string, so a file's contents can never be parsed as code. + /// The cost is `ARG_MAX`: a file larger than roughly a megabyte needs a different transport, + /// and fails loudly here rather than being silently truncated. + async fn write_files(&self, session_id: &str, files: BTreeMap>) -> Result<()> { + for (path, contents) in files { + let path = self.checked_path(&path, "sandbox.writeFiles")?; + let encoded = BASE64.encode(&contents); + + let command = vec![ + "/bin/sh".to_string(), + "-c".to_string(), + // Parent directories are created, matching the in-sandbox agent, so one path + // means the same thing on every backend. + "mkdir -p \"$(dirname \"$2\")\" && printf %s \"$1\" | base64 -d > \"$2\"".to_string(), + "sh".to_string(), + encoded, + path, + ]; + + self.control( + "sandbox.writeFiles", + &self.exec_arguments(session_id, &command), + ) + .await?; + } + + Ok(()) + } + + async fn mkdir(&self, session_id: &str, path: &str) -> Result<()> { + let path = self.checked_path(path, "sandbox.mkdir")?; + let command = vec!["/bin/mkdir".to_string(), "-p".to_string(), path]; + self.control("sandbox.mkdir", &self.exec_arguments(session_id, &command)) + .await?; + Ok(()) + } + + async fn preview(&self, _session_id: &str, _port: u16) -> Result { + Err(self.unsupported( + "preview", + "a Cloud Run sandbox has no ingress of its own and no addressable endpoint", + )) + } + + async fn suspend(&self, _session_id: &str) -> Result<()> { + Err(self.unsupported("suspendResume", "the launcher has no suspend verb")) + } + + async fn resume(&self, _session_id: &str) -> Result<()> { + Err(self.unsupported("suspendResume", "the launcher has no resume verb")) + } + + async fn snapshot(&self, _session_id: &str) -> Result { + Err(self.unsupported( + "snapshot", + "`sandbox fork` produces another live sandbox rather than a durable artifact", + )) + } + + async fn terminate(&self, session_id: &str) -> Result<()> { + self.control( + "sandbox.terminate", + &["delete".to_string(), session_id.to_string()], + ) + .await?; + Ok(()) + } +} + +impl From for CommandOutput { + fn from(frame: ProcessFrame) -> Self { + match frame { + ProcessFrame::Output { + seq, + stream: ProcessStream::Stdout, + data, + } => CommandOutput::Stdout { seq, data }, + ProcessFrame::Output { + seq, + stream: ProcessStream::Stderr, + data, + } => CommandOutput::Stderr { seq, data }, + ProcessFrame::Exit { code, truncated } => CommandOutput::Exit { code, truncated }, + // Handled as a stream error before it reaches here, because an exit code would + // claim the command reported something it never did. + ProcessFrame::Failed { code, message } => { + unreachable!("a failed frame is mapped to an error: {code} {message}") + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use alien_core::bindings::BindingValue; + + /// A fake launcher: it records the argv it was given and answers like the real one. + /// + /// Testing against a script rather than a mock is deliberate. What this provider gets wrong + /// is argument construction, and a mock of the launcher would be built from the same + /// misunderstanding as the code. + fn launcher(body: &str) -> (tempfile::TempDir, GcpSandbox) { + let directory = tempfile::tempdir().expect("temp dir"); + let path = directory.path().join("sandbox"); + std::fs::write(&path, format!("#!/bin/sh\n{body}\n")).expect("write launcher"); + + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o755)) + .expect("make executable"); + } + + let sandbox = GcpSandbox::new( + "sbx", + &GcpSandboxBinding { + launcher_path: BindingValue::value(path.display().to_string()), + allow_egress: BindingValue::value(false), + }, + ) + .expect("binding is valid"); + + (directory, sandbox) + } + + #[tokio::test] + async fn create_names_the_session_and_withholds_egress() { + let (_dir, sandbox) = launcher(r#"echo "$@""#); + + let session = sandbox + .create(CreateSessionRequest { + session_id: Some("s1".to_string()), + tenant_key: None, + env: BTreeMap::new(), + }) + .await + .expect("create succeeds"); + + assert_eq!(session.session_id, "s1"); + assert_eq!(session.state, SandboxSessionState::Running); + } + + /// The launcher takes `--allow-egress` per sandbox, so an application that could pass its + /// own would choose its own confinement. The binding decides it. + #[tokio::test] + async fn egress_comes_from_the_binding_and_not_from_the_request() { + let (dir, _) = launcher(r#"echo "$@" > "$(dirname "$0")/argv""#); + let path = dir.path().join("sandbox"); + + for (allow, expected) in [(false, false), (true, true)] { + let sandbox = GcpSandbox::new( + "sbx", + &GcpSandboxBinding { + launcher_path: BindingValue::value(path.display().to_string()), + allow_egress: BindingValue::value(allow), + }, + ) + .expect("binding is valid"); + + sandbox + .create(CreateSessionRequest { + session_id: Some("s1".to_string()), + tenant_key: None, + env: BTreeMap::new(), + }) + .await + .expect("create succeeds"); + + let argv = std::fs::read_to_string(dir.path().join("argv")).expect("argv recorded"); + assert_eq!( + argv.contains("--allow-egress"), + expected, + "binding said allow_egress={allow}, argv was: {argv}" + ); + } + } + + /// A launcher that fails must not report a session. The cause is on stderr, and losing it + /// turns a specific failure into a guess. + #[tokio::test] + async fn a_failing_launcher_surfaces_its_stderr() { + let (_dir, sandbox) = launcher(r#"echo "quota exhausted" 1>&2; exit 7"#); + + let error = sandbox + .create(CreateSessionRequest { + session_id: Some("s1".to_string()), + tenant_key: None, + env: BTreeMap::new(), + }) + .await + .expect_err("a non-zero launcher exit is a failure"); + + let rendered = format!("{error:?}"); + assert!(rendered.contains("quota exhausted"), "got: {rendered}"); + assert!(rendered.contains('7'), "the exit code belongs in the error: {rendered}"); + } + + #[tokio::test] + async fn a_command_streams_output_and_a_real_exit_code() { + let (_dir, sandbox) = launcher(r#"echo hello; echo problem 1>&2; exit 3"#); + + let frames: Vec<_> = sandbox + .run_command( + "s1", + RunCommandRequest { + command: vec!["/bin/true".to_string()], + working_directory: None, + env: BTreeMap::new(), + deadline: Duration::from_secs(10), + }, + ) + .await + .expect("the command runs") + .collect() + .await; + + let decoded: String = frames + .iter() + .filter_map(|frame| match frame { + Ok(CommandOutput::Stdout { data, .. }) => Some(String::from_utf8_lossy(data).to_string()), + _ => None, + }) + .collect(); + assert!(decoded.contains("hello"), "stdout was: {decoded}"); + + assert!( + frames.iter().any(|frame| matches!( + frame, + Ok(CommandOutput::Stderr { .. }) + )), + "stderr must be framed, not dropped" + ); + + assert!( + matches!(frames.last(), Some(Ok(CommandOutput::Exit { code: 3, .. }))), + "the terminal frame must carry the real exit code: {:?}", + frames.last() + ); + } + + /// Declared capabilities and actual behaviour have to agree, or a caller branches on a lie. + #[tokio::test] + async fn unsupported_capabilities_error_rather_than_pretend() { + let (_dir, sandbox) = launcher("exit 0"); + let capabilities = sandbox.capabilities(); + + assert!(!capabilities.reconnect); + assert!(!capabilities.preview); + assert!(!capabilities.suspend_resume); + assert!(!capabilities.snapshot); + + sandbox.get("s1").await.expect_err("reconnect is not offered"); + sandbox.list().await.expect_err("enumeration is not offered"); + sandbox.preview("s1", 8080).await.expect_err("preview is not offered"); + sandbox.suspend("s1").await.expect_err("suspend is not offered"); + sandbox.resume("s1").await.expect_err("resume is not offered"); + sandbox.snapshot("s1").await.expect_err("snapshot is not offered"); + } + + /// The lexical rule the agent applies, applied here too, so one path means one thing. + #[tokio::test] + async fn a_traversing_path_is_refused_before_the_launcher_sees_it() { + let (_dir, sandbox) = launcher("exit 0"); + + sandbox + .read_file("s1", "../etc/passwd") + .await + .expect_err("a traversing path must be refused"); + sandbox + .write_files( + "s1", + BTreeMap::from([("../etc/passwd".to_string(), b"x".to_vec())]), + ) + .await + .expect_err("a traversing path must be refused on write too"); + } +} diff --git a/crates/alien-bindings/src/providers/sandbox/kubernetes.rs b/crates/alien-bindings/src/providers/sandbox/kubernetes.rs new file mode 100644 index 000000000..1d1fea01d --- /dev/null +++ b/crates/alien-bindings/src/providers/sandbox/kubernetes.rs @@ -0,0 +1,349 @@ +//! Kubernetes sandbox provider: a pod under a sandboxed runtime class, reached over the agent +//! protocol. +//! +//! The application never holds a cluster credential. It asks the operator's broker for a +//! session, and gets back a pod address plus a capability scoped to that session. Claiming a pod +//! is a `PATCH` on pods, which does not belong in the binding: `pods/exec` +//! would reach every pod in the namespace. +//! +//! It authenticates to the broker with the ServiceAccount token Kubernetes already mounted in +//! its pod. Nothing of Alien's is created, rotated or torn down for this. + +use std::collections::BTreeMap; +use std::sync::Mutex; + +use async_trait::async_trait; +use futures::stream::BoxStream; +use serde::{Deserialize, Serialize}; + +use crate::error::{ErrorData, Result}; +use crate::providers::sandbox::agent_protocol::{self, AgentTransport}; +use crate::traits::{ + Binding, CommandOutput, CreateSessionRequest, PreviewCapability, RunCommandRequest, Sandbox, + SandboxSession, SandboxSessionState, +}; +use alien_core::bindings::KubernetesSandboxBinding; +use alien_core::{Platform, SandboxCapabilities}; +use alien_error::{AlienError, Context, IntoAlienError}; + +/// What the broker hands back for a claimed session. +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +struct ClaimResponse { + session_id: String, + endpoint: String, + capability: String, + expires_at: i64, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct ClaimRequest<'a> { + sandbox_id: &'a str, + session_id: &'a str, +} + +/// A Sandbox backed by pods under a sandboxed runtime class. +#[derive(Debug)] +pub struct KubernetesSandbox { + sandbox_id: String, + broker_url: String, + token_path: String, + binding_name: String, + client: reqwest::Client, + /// Claims this process has made, so a later call can address the session it already has. + /// + /// The capability is short-lived and the endpoint is a pod IP, so this is a cache of live + /// sessions rather than durable state. A session this process did not claim is not + /// reachable, which is what `reconnect` means here. + claims: Mutex>, +} + +impl KubernetesSandbox { + /// Builds a provider from its binding. + pub fn new(binding_name: &str, binding: &KubernetesSandboxBinding, sandbox_id: &str) -> Result { + let value = |field: &'static str, value: alien_core::bindings::BindingValue| { + value.into_value(binding_name, field).map_err(|error| { + AlienError::new(ErrorData::BindingConfigInvalid { + binding_name: binding_name.to_string(), + env_var: alien_core::bindings::binding_env_var_name(binding_name), + reason: error.to_string(), + }) + }) + }; + + Ok(Self { + sandbox_id: sandbox_id.to_string(), + broker_url: value("brokerUrl", binding.broker_url.clone())? + .trim_end_matches('/') + .to_string(), + token_path: value("tokenPath", binding.token_path.clone())?, + binding_name: binding_name.to_string(), + client: reqwest::Client::new(), + claims: Mutex::new(BTreeMap::new()), + }) + } + + /// Reads the pod's ServiceAccount token. + /// + /// Read per call rather than cached: Kubernetes rotates projected tokens in place, and a + /// cached copy becomes a token the apiserver refuses at the least convenient moment. + async fn identity_token(&self) -> Result { + tokio::fs::read_to_string(&self.token_path) + .await + .into_alien_error() + .context(ErrorData::BindingConfigInvalid { + binding_name: self.binding_name.clone(), + env_var: alien_core::bindings::binding_env_var_name(&self.binding_name), + reason: format!( + "could not read the ServiceAccount token at '{}'", + self.token_path + ), + }) + } + + fn claimed(&self, session_id: &str) -> Option { + self.claims + .lock() + .expect("no panic holds this lock") + .get(session_id) + .cloned() + } + + fn failed(&self, operation: &str, reason: &str) -> AlienError { + AlienError::new(ErrorData::OperationNotSupported { + operation: operation.to_string(), + reason: reason.to_string(), + }) + } +} + +#[async_trait] +impl AgentTransport for KubernetesSandbox { + async fn request( + &self, + session_id: &str, + method: reqwest::Method, + path: &str, + ) -> Result { + let claim = self.claimed(session_id).ok_or_else(|| { + self.failed( + "sandbox.agent", + &format!( + "session '{session_id}' was not claimed by this process; a pod IP and a \ + capability are only reachable by the caller that claimed them" + ), + ) + })?; + + if claim.expires_at <= chrono::Utc::now().timestamp() { + return Err(self.failed( + "sandbox.agent", + &format!( + "the capability for session '{session_id}' expired; the agent would refuse \ + this with a 401 that reads like a broken sandbox" + ), + )); + } + + Ok(self + .client + .request(method, format!("{}{path}", claim.endpoint)) + .bearer_auth(claim.capability)) + } + + fn provider(&self) -> &'static str { + "kubernetes-sandbox" + } +} + +impl Binding for KubernetesSandbox {} + +#[async_trait] +impl Sandbox for KubernetesSandbox { + fn as_any(&self) -> &dyn std::any::Any { + self + } + + fn capabilities(&self) -> SandboxCapabilities { + SandboxCapabilities::for_platform(Platform::Kubernetes) + .expect("Kubernetes has a sandbox backend") + } + + /// Claims a warm pod through the broker. + async fn create(&self, request: CreateSessionRequest) -> Result { + let session_id = request + .session_id + .unwrap_or_else(|| uuid::Uuid::new_v4().simple().to_string()); + + let response = self + .client + .post(format!("{}/v1/sandbox/sessions", self.broker_url)) + .bearer_auth(self.identity_token().await?) + .json(&ClaimRequest { + sandbox_id: &self.sandbox_id, + session_id: &session_id, + }) + .send() + .await + .into_alien_error() + .context(ErrorData::OperationNotSupported { + operation: "sandbox.create".to_string(), + reason: "the sandbox broker is unreachable".to_string(), + })?; + + if !response.status().is_success() { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + // 503 is the pool being empty, which refills on the controller's next health tick. + // Saying so is the difference between a caller retrying and a caller giving up. + return Err(self.failed( + "sandbox.create", + &format!("the sandbox broker returned {status}: {body}"), + )); + } + + let claim: ClaimResponse = + response + .json() + .await + .into_alien_error() + .context(ErrorData::UnexpectedResponseFormat { + provider: "kubernetes-sandbox".to_string(), + binding_name: "sandbox.create".to_string(), + field: "body".to_string(), + response_json: "the broker returned a body this provider cannot parse" + .to_string(), + })?; + + self.claims + .lock() + .expect("no panic holds this lock") + .insert(claim.session_id.clone(), claim.clone()); + + Ok(SandboxSession { + session_id: claim.session_id, + state: SandboxSessionState::Running, + // A released pod is deleted rather than fenced, so a session never outlives its own + // generation. + generation: 1, + }) + } + + /// Only sessions this process claimed are addressable. + /// + /// A capability is minted to the caller that claimed the pod, so another process holding the + /// same session id has nothing to reach it with. Returning `None` rather than erroring: the + /// session may well exist, this caller simply cannot address it. + async fn get(&self, session_id: &str) -> Result> { + Ok(self.claimed(session_id).map(|claim| SandboxSession { + session_id: claim.session_id, + state: SandboxSessionState::Running, + generation: 1, + })) + } + + async fn get_or_create(&self, request: CreateSessionRequest) -> Result { + if let Some(id) = request.session_id.as_deref() { + if let Some(existing) = self.get(id).await? { + return Ok(existing); + } + } + + self.create(request).await + } + + async fn list(&self) -> Result> { + Ok(self + .claims + .lock() + .expect("no panic holds this lock") + .values() + .map(|claim| SandboxSession { + session_id: claim.session_id.clone(), + state: SandboxSessionState::Running, + generation: 1, + }) + .collect()) + } + + async fn run_command( + &self, + session_id: &str, + request: RunCommandRequest, + ) -> Result>> { + agent_protocol::run_command(self, session_id, request).await + } + + async fn read_file(&self, session_id: &str, path: &str) -> Result> { + agent_protocol::read_file(self, session_id, path).await + } + + async fn write_files(&self, session_id: &str, files: BTreeMap>) -> Result<()> { + agent_protocol::write_files(self, session_id, files).await + } + + async fn mkdir(&self, session_id: &str, path: &str) -> Result<()> { + agent_protocol::mkdir(self, session_id, path).await + } + + async fn preview(&self, _session_id: &str, _port: u16) -> Result { + Err(self.failed( + "preview", + "preview needs a gateway that validates a session-and-port capability, and that \ + gateway does not exist yet", + )) + } + + async fn suspend(&self, _session_id: &str) -> Result<()> { + Err(self.failed("suspendResume", "a pod cannot be suspended and resumed")) + } + + async fn resume(&self, _session_id: &str) -> Result<()> { + Err(self.failed("suspendResume", "a pod cannot be suspended and resumed")) + } + + async fn snapshot(&self, _session_id: &str) -> Result { + Err(self.failed("snapshot", "a pod has no snapshot primitive")) + } + + /// Releases the session, which deletes its pod. + /// + /// Idempotent: a session this process never claimed is already in the desired end state. + async fn terminate(&self, session_id: &str) -> Result<()> { + let Some(claim) = self.claimed(session_id) else { + return Ok(()); + }; + + let response = self + .client + .delete(format!( + "{}/v1/sandbox/{}/sessions/{}", + self.broker_url, self.sandbox_id, claim.session_id + )) + .bearer_auth(self.identity_token().await?) + .send() + .await + .into_alien_error() + .context(ErrorData::OperationNotSupported { + operation: "sandbox.terminate".to_string(), + reason: "the sandbox broker is unreachable".to_string(), + })?; + + if !response.status().is_success() { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + return Err(self.failed( + "sandbox.terminate", + &format!("the sandbox broker returned {status}: {body}"), + )); + } + + self.claims + .lock() + .expect("no panic holds this lock") + .remove(session_id); + + Ok(()) + } +} diff --git a/crates/alien-bindings/src/providers/sandbox/local.rs b/crates/alien-bindings/src/providers/sandbox/local.rs new file mode 100644 index 000000000..c0e2463cf --- /dev/null +++ b/crates/alien-bindings/src/providers/sandbox/local.rs @@ -0,0 +1,428 @@ +//! Local sandbox provider. +//! +//! Speaks to the local sandbox manager over its authenticated loopback route. It cannot call +//! the manager in process — `alien-local` depends on this crate, so a direct call would be a +//! dependency cycle — and handing the workload a Docker socket instead would give every +//! application the ability to escape its own sandbox. + +use std::collections::BTreeMap; + +use async_trait::async_trait; +use base64::engine::general_purpose::STANDARD as BASE64; +use base64::Engine as _; +use futures::stream::{self, BoxStream}; +use futures::StreamExt; +use serde::Deserialize; +use serde_json::json; + +use crate::error::{ErrorData, Result}; +use crate::traits::{ + Binding, CommandOutput, CreateSessionRequest, PreviewCapability, RunCommandRequest, Sandbox, + SandboxSession, SandboxSessionState, +}; +use alien_core::bindings::LocalSandboxBinding; +use alien_core::{Platform, SandboxCapabilities}; +use alien_error::{AlienError, Context, IntoAlienError}; + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct SessionBody { + session_id: String, + #[allow(dead_code)] + container_id: String, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase", tag = "stream", content = "dataBase64")] +enum OutputFrame { + Stdout(String), + Stderr(String), +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct ExecResponse { + output: Vec, + exit_code: i64, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct ReadFileResponse { + contents_base64: String, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct PreviewResponse { + endpoint: String, + allowed_ports: Vec, +} + +/// A Sandbox backed by the local manager. +#[derive(Debug)] +pub struct LocalSandbox { + client: reqwest::Client, + base_url: String, + token: String, +} + +impl LocalSandbox { + /// Builds a provider from its binding, reading the route token from the path it names. + /// + /// The binding carries a path rather than the token itself: a binding is serialized into + /// the workload's environment, and a secret there is a secret in state. + pub async fn new(binding_name: &str, binding: &LocalSandboxBinding) -> Result { + let base_url = binding + .manager_url + .clone() + .into_value(binding_name, "managerUrl") + .map_err(|error| { + AlienError::new(ErrorData::BindingConfigInvalid { + binding_name: binding_name.to_string(), + env_var: alien_core::bindings::binding_env_var_name(binding_name), + reason: error.to_string(), + }) + })?; + + let token_path = binding + .token_path + .clone() + .into_value(binding_name, "tokenPath") + .map_err(|error| { + AlienError::new(ErrorData::BindingConfigInvalid { + binding_name: binding_name.to_string(), + env_var: alien_core::bindings::binding_env_var_name(binding_name), + reason: error.to_string(), + }) + })?; + + let token = tokio::fs::read_to_string(&token_path) + .await + .into_alien_error() + .context(ErrorData::BindingConfigInvalid { + binding_name: binding_name.to_string(), + env_var: alien_core::bindings::binding_env_var_name(binding_name), + reason: format!("could not read the sandbox route token at '{token_path}'"), + })?; + + Ok(Self { + client: reqwest::Client::new(), + base_url: base_url.trim_end_matches('/').to_string(), + token: token.trim().to_string(), + }) + } + + fn url(&self, path: &str) -> String { + format!("{}{path}", self.base_url) + } + + async fn send Deserialize<'de>>( + &self, + request: reqwest::RequestBuilder, + operation: &str, + ) -> Result { + let response = self + .request(request, operation) + .await? + .json::() + .await + .into_alien_error() + .context(ErrorData::UnexpectedResponseFormat { + provider: "local-sandbox".to_string(), + binding_name: operation.to_string(), + field: "body".to_string(), + response_json: "the sandbox route returned a body this provider cannot parse" + .to_string(), + })?; + + Ok(response) + } + + async fn request( + &self, + request: reqwest::RequestBuilder, + operation: &str, + ) -> Result { + let response = request + .bearer_auth(&self.token) + .send() + .await + .into_alien_error() + .context(ErrorData::OperationNotSupported { + operation: operation.to_string(), + reason: "the local sandbox route is unreachable".to_string(), + })?; + + if response.status().is_success() { + return Ok(response); + } + + let status = response.status(); + // Read the body before reporting: the route puts the actual cause there, and a bare + // status turns a specific failure into a guess. + let body = response.text().await.unwrap_or_default(); + Err(AlienError::new(ErrorData::OperationNotSupported { + operation: operation.to_string(), + reason: format!("the local sandbox route returned {status}: {body}"), + })) + } + + fn unsupported(&self, capability: &str) -> AlienError { + AlienError::new(ErrorData::OperationNotSupported { + operation: capability.to_string(), + reason: "not supported on local".to_string(), + }) + } +} + +impl Binding for LocalSandbox {} + +#[async_trait] +impl Sandbox for LocalSandbox { + fn capabilities(&self) -> SandboxCapabilities { + SandboxCapabilities::for_platform(Platform::Local) + .expect("Local has a sandbox backend") + } + + async fn create(&self, request: CreateSessionRequest) -> Result { + let session_id = request + .session_id + .unwrap_or_else(|| uuid::Uuid::new_v4().simple().to_string()); + + // Only the id. Image, limits, egress and preview ports come from the controller's + // template, so an application cannot raise its own ceilings. + let created: SessionBody = self + .send( + self.client + .post(self.url("/v1/sessions")) + .json(&json!({ "sessionId": session_id })), + "sandbox.create", + ) + .await?; + + Ok(SandboxSession { + session_id: created.session_id, + state: SandboxSessionState::Running, + generation: 1, + }) + } + + async fn get(&self, session_id: &str) -> Result> { + Ok(self + .list() + .await? + .into_iter() + .find(|session| session.session_id == session_id)) + } + + async fn get_or_create(&self, request: CreateSessionRequest) -> Result { + if let Some(id) = request.session_id.as_deref() { + if let Some(existing) = self.get(id).await? { + return Ok(existing); + } + } + + self.create(request).await + } + + async fn list(&self) -> Result> { + let sessions: Vec = self + .send(self.client.get(self.url("/v1/sessions")), "sandbox.list") + .await?; + + Ok(sessions + .into_iter() + .map(|session| SandboxSession { + session_id: session.session_id, + state: SandboxSessionState::Running, + generation: 1, + }) + .collect()) + } + + async fn run_command( + &self, + session_id: &str, + request: RunCommandRequest, + ) -> Result>> { + if request.deadline.is_zero() { + return Err(AlienError::new(ErrorData::OperationNotSupported { + operation: "sandbox.runCommand".to_string(), + reason: "a command must carry a non-zero deadline".to_string(), + })); + } + + // The deadline bounds the untrusted code, not the caller's patience. The route runs the + // command to completion, so the only lever that actually stops one past its ceiling is + // ending the session — a command that overran took the session with it. + let response: ExecResponse = match tokio::time::timeout( + request.deadline, + self.send( + self.client + .post(self.url(&format!("/v1/sessions/{session_id}/exec"))) + .json(&json!({ "command": request.command })), + "sandbox.runCommand", + ), + ) + .await + { + Ok(inner) => inner?, + Err(_) => { + self.terminate(session_id).await?; + return Err(AlienError::new(ErrorData::SandboxCommandFailed { + failure: "deadlineExceeded".to_string(), + reason: format!( + "the command exceeded its {}s deadline and the session was terminated", + request.deadline.as_secs() + ), + })); + } + }; + + let mut frames: Vec> = Vec::new(); + for (index, frame) in response.output.into_iter().enumerate() { + let seq = index as u64; + let decoded = match &frame { + OutputFrame::Stdout(data) | OutputFrame::Stderr(data) => BASE64 + .decode(data) + .into_alien_error() + .context(ErrorData::UnexpectedResponseFormat { + provider: "local-sandbox".to_string(), + binding_name: "sandbox.runCommand".to_string(), + field: "output".to_string(), + response_json: "an output frame was not valid base64".to_string(), + })?, + }; + + frames.push(Ok(match frame { + OutputFrame::Stdout(_) => CommandOutput::Stdout { + seq, + data: decoded, + }, + OutputFrame::Stderr(_) => CommandOutput::Stderr { + seq, + data: decoded, + }, + })); + } + + frames.push(Ok(CommandOutput::Exit { + code: response.exit_code as i32, + truncated: false, + })); + + Ok(Box::pin(stream::iter(frames))) + } + + async fn read_file(&self, session_id: &str, path: &str) -> Result> { + let response: ReadFileResponse = self + .send( + self.client + .get(self.url(&format!("/v1/sessions/{session_id}/files"))) + .query(&[("path", path)]), + "sandbox.readFile", + ) + .await?; + + BASE64 + .decode(response.contents_base64) + .into_alien_error() + .context(ErrorData::UnexpectedResponseFormat { + provider: "local-sandbox".to_string(), + binding_name: "sandbox.readFile".to_string(), + field: "contentsBase64".to_string(), + response_json: "file contents were not valid base64".to_string(), + }) + } + + async fn write_files(&self, session_id: &str, files: BTreeMap>) -> Result<()> { + for (path, contents) in files { + self.request( + self.client + .put(self.url(&format!("/v1/sessions/{session_id}/files"))) + .json(&json!({ + "path": path, + "contentsBase64": BASE64.encode(contents), + })), + "sandbox.writeFiles", + ) + .await?; + } + + Ok(()) + } + + async fn mkdir(&self, session_id: &str, path: &str) -> Result<()> { + let request = RunCommandRequest { + command: vec!["/bin/mkdir".to_string(), "-p".to_string(), path.to_string()], + working_directory: None, + env: BTreeMap::new(), + deadline: std::time::Duration::from_secs(30), + }; + + // Drain to the terminal frame: the command has already run by the time the stream is + // built, but a non-zero exit means the directory does not exist and the caller must hear + // about it rather than discover it on the next write. + let mut frames = self.run_command(session_id, request).await?; + while let Some(frame) = frames.next().await { + if let CommandOutput::Exit { code, .. } = frame? { + if code != 0 { + return Err(AlienError::new(ErrorData::OperationNotSupported { + operation: "sandbox.mkdir".to_string(), + reason: format!("mkdir '{path}' exited with {code}"), + })); + } + } + } + + Ok(()) + } + + async fn preview(&self, session_id: &str, port: u16) -> Result { + let response: PreviewResponse = self + .send( + self.client + .get(self.url(&format!("/v1/sessions/{session_id}/preview"))) + .query(&[("port", port.to_string())]), + "sandbox.preview", + ) + .await?; + + Ok(PreviewCapability { + endpoint: response.endpoint, + // The port is published on loopback, so reaching it needs no credential beyond + // being on the developer's machine. Stated rather than implied by an empty map. + headers: BTreeMap::new(), + allowed_ports: response.allowed_ports, + expires_in_seconds: 0, + }) + } + + async fn suspend(&self, _session_id: &str) -> Result<()> { + Err(self.unsupported("suspendResume")) + } + + async fn resume(&self, _session_id: &str) -> Result<()> { + Err(self.unsupported("suspendResume")) + } + + async fn snapshot(&self, _session_id: &str) -> Result { + Err(self.unsupported("snapshot")) + } + + async fn terminate(&self, session_id: &str) -> Result<()> { + self.request( + self.client + .delete(self.url(&format!("/v1/sessions/{session_id}"))), + "sandbox.terminate", + ) + .await?; + + Ok(()) + } + + fn as_any(&self) -> &dyn std::any::Any { + self + } +} diff --git a/crates/alien-bindings/src/providers/sandbox/mod.rs b/crates/alien-bindings/src/providers/sandbox/mod.rs new file mode 100644 index 000000000..de05eb448 --- /dev/null +++ b/crates/alien-bindings/src/providers/sandbox/mod.rs @@ -0,0 +1,24 @@ +//! Sandbox binding providers. +//! +//! Per-cloud backends land with their controllers. Local is here because it speaks the same +//! authenticated transport the cloud backends do, so it exercises the real path rather than a +//! shortcut. + +#[cfg(any(feature = "aws", feature = "kubernetes"))] +pub mod agent_protocol; + +#[cfg(feature = "aws")] +pub mod aws; + +#[cfg(feature = "azure")] +pub mod azure; + +#[cfg(feature = "gcp")] +pub mod gcp; + +#[cfg(feature = "kubernetes")] +pub mod kubernetes; + +#[cfg(feature = "local")] +pub mod local; + diff --git a/crates/alien-bindings/src/traits.rs b/crates/alien-bindings/src/traits.rs index 1bc8ea2bf..df1ecd007 100644 --- a/crates/alien-bindings/src/traits.rs +++ b/crates/alien-bindings/src/traits.rs @@ -1178,7 +1178,8 @@ pub trait BindingsProviderApi: Send + Sync + std::fmt::Debug { /// Given a binding identifier, builds a ServiceAccount implementation. async fn load_service_account(&self, binding_name: &str) -> Result>; - + /// Given a binding identifier, builds a Sandbox implementation. + async fn load_sandbox(&self, binding_name: &str) -> Result>; /// Runtime-only binding env vars (a local Postgres connection with its password, a local /// BYO-key AI binding) for the given resource — re-resolved on every (re)start so the secret diff --git a/crates/alien-core/src/bin/schema_exporter.rs b/crates/alien-core/src/bin/schema_exporter.rs index cba8739e1..6685cc692 100644 --- a/crates/alien-core/src/bin/schema_exporter.rs +++ b/crates/alien-core/src/bin/schema_exporter.rs @@ -192,6 +192,7 @@ use utoipa::OpenApi; AwsArtifactRegistryImportData, AwsComputeClusterImportData, AwsPostgresImportData, + AwsSandboxImportData, GcpStorageImportData, GcpWorkerImportData, GcpQueueImportData, @@ -224,6 +225,7 @@ use utoipa::OpenApi; AzureServiceBusNamespaceImportData, AzureStorageAccountImportData, AzureFlexibleServerPostgresImportData, + AzureSandboxImportData, )))] struct ApiDoc; diff --git a/crates/alien-core/src/import/data/aws/mod.rs b/crates/alien-core/src/import/data/aws/mod.rs index 44a604394..dfaa96f0d 100644 --- a/crates/alien-core/src/import/data/aws/mod.rs +++ b/crates/alien-core/src/import/data/aws/mod.rs @@ -7,6 +7,7 @@ pub mod kv; pub mod network; pub mod open_search; pub mod postgres; +pub mod sandbox; pub mod queue; pub mod remote_bindings; pub mod remote_stack_management; @@ -24,6 +25,7 @@ pub use kv::*; pub use network::*; pub use open_search::*; pub use postgres::*; +pub use sandbox::*; pub use queue::*; pub use remote_bindings::*; pub use remote_stack_management::*; diff --git a/crates/alien-core/src/import/data/aws/sandbox.rs b/crates/alien-core/src/import/data/aws/sandbox.rs new file mode 100644 index 000000000..3b7f1635a --- /dev/null +++ b/crates/alien-core/src/import/data/aws/sandbox.rs @@ -0,0 +1,30 @@ +use serde::{Deserialize, Serialize}; + +/// AWS Sandbox ImportData. +/// +/// Carries the Frozen parent from the setup emitter to the runtime controller. The image +/// **version** is not decoration: `RunMicrovm` has no `tags`, so image plus version is the only +/// session identity there is, and a controller holding a stale version would enumerate the wrong +/// set and orphan every session started on the previous one. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))] +#[cfg_attr(feature = "jsonschema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase")] +pub struct AwsSandboxImportData { + /// MicroVM image identifier. + pub image_identifier: String, + /// MicroVM image ARN. + pub image_arn: String, + /// Image version the sessions are scoped to. Re-imported on every image roll. + pub image_version: String, + /// Execution role attached to each MicroVM, distinct from the workload's own role. + #[serde(skip_serializing_if = "Option::is_none")] + pub execution_role_arn: Option, + /// Egress network connectors. Deleting one while MicroVMs still reference it breaks their + /// networking, so teardown needs them named rather than rediscovered. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub egress_connector_arns: Vec, + /// Ports a preview capability may be minted for; empty means preview is not offered. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub preview_ports: Vec, +} diff --git a/crates/alien-core/src/import/data/azure/mod.rs b/crates/alien-core/src/import/data/azure/mod.rs index 4d8b7cbb2..0370d00e7 100644 --- a/crates/alien-core/src/import/data/azure/mod.rs +++ b/crates/alien-core/src/import/data/azure/mod.rs @@ -13,6 +13,7 @@ pub mod resource_group; pub mod service_account; pub mod service_activation; pub mod service_bus_namespace; +pub mod sandbox; pub mod storage; pub mod storage_account; pub mod vault; @@ -33,6 +34,7 @@ pub use resource_group::*; pub use service_account::*; pub use service_activation::*; pub use service_bus_namespace::*; +pub use sandbox::*; pub use storage::*; pub use storage_account::*; pub use vault::*; diff --git a/crates/alien-core/src/import/data/azure/sandbox.rs b/crates/alien-core/src/import/data/azure/sandbox.rs new file mode 100644 index 000000000..981e22116 --- /dev/null +++ b/crates/alien-core/src/import/data/azure/sandbox.rs @@ -0,0 +1,19 @@ +use serde::{Deserialize, Serialize}; + +/// Azure Sandbox ImportData. +/// +/// Carries the sandbox group from the setup emitter to the runtime controller. All three fields +/// are required to address it: the ADC data plane endpoint is **per-region**, so a group without +/// its region cannot be reached at all, and the data plane path is scoped by resource group. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "openapi", derive(utoipa::ToSchema))] +#[cfg_attr(feature = "jsonschema", derive(schemars::JsonSchema))] +#[serde(rename_all = "camelCase")] +pub struct AzureSandboxImportData { + /// Sandbox group name. + pub sandbox_group: String, + /// Region the group lives in; selects the ADC endpoint. + pub region: String, + /// Resource group containing the sandbox group. + pub resource_group: String, +} diff --git a/crates/alien-core/src/import/data/mod.rs b/crates/alien-core/src/import/data/mod.rs index 01b902caf..95d4c35aa 100644 --- a/crates/alien-core/src/import/data/mod.rs +++ b/crates/alien-core/src/import/data/mod.rs @@ -11,6 +11,7 @@ pub use aws::{ AwsAiImportData, AwsArtifactRegistryImportData, AwsBuildImportData, AwsComputeClusterImportData, AwsEmailDkimTokenImportData, AwsEmailDomainImportData, AwsEmailImportData, AwsKvImportData, AwsNetworkImportData, AwsOpenSearchImportData, + AwsSandboxImportData, AwsPostgresImportData, AwsQueueImportData, AwsRemoteBindingsImportData, AwsRemoteStackManagementImportData, AwsServiceAccountImportData, AwsStorageImportData, AwsVaultImportData, AwsWorkerImportData, @@ -19,6 +20,7 @@ pub use azure::{ AzureAiImportData, AzureArtifactRegistryImportData, AzureBuildImportData, AzureComputeClusterImportData, AzureContainerAppsEnvironmentImportData, AzureFlexibleServerPostgresImportData, AzureKvImportData, AzureNetworkImportData, + AzureSandboxImportData, AzureQueueImportData, AzureRemoteBindingsImportData, AzureRemoteStackManagementImportData, AzureResourceGroupImportData, AzureServiceAccountImportData, AzureServiceActivationImportData, AzureServiceBusNamespaceImportData, AzureStorageAccountImportData, AzureStorageImportData, @@ -77,6 +79,7 @@ mod schema_snapshots { ("aws_email", schema::()), ("aws_function", schema::()), ("aws_kv", schema::()), + ("aws_sandbox", schema::()), ("aws_network", schema::()), ("aws_open_search", schema::()), ("aws_postgres", schema::()), @@ -107,6 +110,7 @@ mod schema_snapshots { ), ("azure_function", schema::()), ("azure_kv", schema::()), + ("azure_sandbox", schema::()), ("azure_network", schema::()), ( "azure_postgres", diff --git a/crates/alien-core/src/import/data/snapshots/alien_core__import__data__schema_snapshots__import_data_schemas.snap b/crates/alien-core/src/import/data/snapshots/alien_core__import__data__schema_snapshots__import_data_schemas.snap index a27492af7..d4afe2e27 100644 --- a/crates/alien-core/src/import/data/snapshots/alien_core__import__data__schema_snapshots__import_data_schemas.snap +++ b/crates/alien-core/src/import/data/snapshots/alien_core__import__data__schema_snapshots__import_data_schemas.snap @@ -303,6 +303,54 @@ expression: schemas "title": "AwsKvImportData", "type": "object" }, + "aws_sandbox": { + "$schema": "http://json-schema.org/draft-07/schema#", + "description": "AWS Sandbox ImportData.\n\nCarries the Frozen parent from the setup emitter to the runtime controller. The image **version** is not decoration: `RunMicrovm` has no `tags`, so image plus version is the only session identity there is, and a controller holding a stale version would enumerate the wrong set and orphan every session started on the previous one.", + "properties": { + "egressConnectorArns": { + "description": "Egress network connectors. Deleting one while MicroVMs still reference it breaks their networking, so teardown needs them named rather than rediscovered.", + "items": { + "type": "string" + }, + "type": "array" + }, + "executionRoleArn": { + "description": "Execution role attached to each MicroVM, distinct from the workload's own role.", + "type": [ + "string", + "null" + ] + }, + "imageArn": { + "description": "MicroVM image ARN.", + "type": "string" + }, + "imageIdentifier": { + "description": "MicroVM image identifier.", + "type": "string" + }, + "imageVersion": { + "description": "Image version the sessions are scoped to. Re-imported on every image roll.", + "type": "string" + }, + "previewPorts": { + "description": "Ports a preview capability may be minted for; empty means preview is not offered.", + "items": { + "format": "uint16", + "minimum": 0.0, + "type": "integer" + }, + "type": "array" + } + }, + "required": [ + "imageArn", + "imageIdentifier", + "imageVersion" + ], + "title": "AwsSandboxImportData", + "type": "object" + }, "aws_network": { "$schema": "http://json-schema.org/draft-07/schema#", "definitions": { @@ -547,6 +595,13 @@ expression: schemas "description": "Whether the management inline policy was attached by the generated stack.", "type": "boolean" }, + "remoteBindingsRoleArn": { + "description": "Setup-owned role used only for opted-in remote binding data access.", + "type": [ + "string", + "null" + ] + }, "roleArn": { "description": "Cross-account management role ARN.", "type": "string" @@ -881,6 +936,31 @@ expression: schemas "title": "AzureKvImportData", "type": "object" }, + "azure_sandbox": { + "$schema": "http://json-schema.org/draft-07/schema#", + "description": "Azure Sandbox ImportData.\n\nCarries the sandbox group from the setup emitter to the runtime controller. All three fields are required to address it: the ADC data plane endpoint is **per-region**, so a group without its region cannot be reached at all, and the data plane path is scoped by resource group.", + "properties": { + "region": { + "description": "Region the group lives in; selects the ADC endpoint.", + "type": "string" + }, + "resourceGroup": { + "description": "Resource group containing the sandbox group.", + "type": "string" + }, + "sandboxGroup": { + "description": "Sandbox group name.", + "type": "string" + } + }, + "required": [ + "region", + "resourceGroup", + "sandboxGroup" + ], + "title": "AzureSandboxImportData", + "type": "object" + }, "azure_network": { "$schema": "http://json-schema.org/draft-07/schema#", "description": "Azure Network ImportData — VNet + subnets + NAT topology.", @@ -1078,6 +1158,20 @@ expression: schemas "description": "Management UAMI principal id.", "type": "string" }, + "remoteBindingsClientId": { + "description": "Remote Bindings UAMI client id used for workload identity exchange.", + "type": [ + "string", + "null" + ] + }, + "remoteBindingsIdentityId": { + "description": "Setup-owned UAMI resource id used only for opted-in remote bindings.", + "type": [ + "string", + "null" + ] + }, "resourceGroup": { "description": "Resource group containing the management identity.", "type": "string" @@ -1733,6 +1827,13 @@ expression: schemas "null" ] }, + "remoteBindingsServiceAccountEmail": { + "description": "Setup-owned service account used only for opted-in remote bindings.", + "type": [ + "string", + "null" + ] + }, "serviceAccountEmail": { "description": "Service account email the manager impersonates.", "type": "string" diff --git a/crates/alien-gcp-clients/src/gcp/cloudrun.rs b/crates/alien-gcp-clients/src/gcp/cloudrun.rs index cd109f01a..08e1f59a6 100644 --- a/crates/alien-gcp-clients/src/gcp/cloudrun.rs +++ b/crates/alien-gcp-clients/src/gcp/cloudrun.rs @@ -1077,6 +1077,14 @@ pub struct Container { #[serde(default, skip_serializing_if = "Vec::is_empty")] pub args: Vec, + /// Lets this container act as a sandbox supervisor and launch sandboxes. + /// + /// The service must also declare `launch_stage: Beta` or later — Cloud Run rejects the + /// field otherwise with `FAILED_PRECONDITION: The feature 'Instant sandboxes' is not + /// supported in the declared launch stage`. + #[serde(skip_serializing_if = "Option::is_none")] + pub sandbox_launcher: Option, + /// List of environment variables to set in the container. #[builder(default)] #[serde(default, skip_serializing_if = "Vec::is_empty")] @@ -1444,6 +1452,39 @@ pub struct BuildInfo { pub source_location: Option, } +#[cfg(test)] +mod sandbox_launcher_tests { + use super::*; + + /// Cloud Run rejects `sandboxLauncher` unless the service declares BETA or later: + /// `FAILED_PRECONDITION: The feature 'Instant sandboxes' is not supported in the declared + /// launch stage`. Verified against the live API, so the two travel together. + #[test] + fn sandbox_launcher_serializes_as_the_api_spells_it() { + let container = Container { + image: "us-docker.pkg.dev/cloudrun/container/hello".to_string(), + sandbox_launcher: Some(true), + ..Default::default() + }; + + let json = serde_json::to_value(&container).expect("serializes"); + assert_eq!(json["sandboxLauncher"], serde_json::json!(true)); + } + + /// Absent rather than `false` when unset, so an ordinary container's request body is + /// unchanged and cannot trip the launch-stage precondition. + #[test] + fn an_ordinary_container_does_not_carry_the_field() { + let container = Container { + image: "img".to_string(), + ..Default::default() + }; + + let json = serde_json::to_value(&container).expect("serializes"); + assert!(json.get("sandboxLauncher").is_none(), "{json}"); + } +} + #[cfg(test)] mod tests { use super::Ingress; diff --git a/crates/alien-infra/Cargo.toml b/crates/alien-infra/Cargo.toml index bfae3530d..3b9430aee 100644 --- a/crates/alien-infra/Cargo.toml +++ b/crates/alien-infra/Cargo.toml @@ -12,7 +12,7 @@ all-platforms = ["aws", "gcp", "azure", "kubernetes", "local", "test"] # Conven aws = ["dep:alien-aws-clients", "dep:alien-preflights", "alien-client-config/aws"] # TODO: Remove preflights from here, necessary because of CF gcp = ["dep:alien-gcp-clients", "alien-client-config/gcp"] azure = ["dep:alien-azure-clients", "alien-client-config/azure", "dep:base64"] -kubernetes = ["dep:alien-k8s-clients", "dep:base64", "dep:dirs", "alien-client-config/kubernetes"] +kubernetes = ["dep:alien-k8s-clients", "dep:axum", "dep:base64", "dep:dirs", "dep:ed25519-compact", "alien-core/sandbox-capability", "alien-client-config/kubernetes"] local = ["dep:alien-local"] # Local platform support test = [] # Test platform support - fast mock implementations without real cloud APIs openapi = ["dep:utoipa", "alien-core/openapi"] @@ -67,7 +67,9 @@ pem = { workspace = true } p12 = "0.6" # Kubeconfig dependencies +axum = { workspace = true, optional = true } base64 = { workspace = true, optional = true } +ed25519-compact = { workspace = true, optional = true } dirs = { workspace = true, optional = true } [target.'cfg(target_arch = "wasm32")'.dependencies] diff --git a/crates/alien-infra/src/core/controller.rs b/crates/alien-infra/src/core/controller.rs index 73cd36a51..8311b5cde 100644 --- a/crates/alien-infra/src/core/controller.rs +++ b/crates/alien-infra/src/core/controller.rs @@ -985,6 +985,12 @@ fn deserialize_controller_by_tag( "LocalComputeClusterController" => { deser!(crate::compute_cluster::LocalComputeClusterController) } + #[cfg(feature = "local")] + "LocalSandboxController" => deser!(crate::sandbox::LocalSandboxController), + #[cfg(feature = "kubernetes")] + "KubernetesSandboxController" => { + deser!(crate::sandbox::KubernetesSandboxController) + } #[cfg(feature = "kubernetes")] "KubernetesClusterController" => { deser!(crate::kubernetes_cluster::KubernetesClusterController) diff --git a/crates/alien-infra/src/core/controller_test.rs b/crates/alien-infra/src/core/controller_test.rs index 99f010f9a..aac543892 100644 --- a/crates/alien-infra/src/core/controller_test.rs +++ b/crates/alien-infra/src/core/controller_test.rs @@ -633,6 +633,7 @@ pub struct SingleControllerExecutorBuilder { public_endpoints: Option, dependencies: Vec<(ResourceRef, Resource, Box)>, service_provider: Option>, + client_config: Option, } impl SingleControllerExecutorBuilder { @@ -655,6 +656,7 @@ impl SingleControllerExecutorBuilder { public_endpoints: None, dependencies: Vec::new(), service_provider: None, + client_config: None, } } @@ -739,6 +741,15 @@ impl SingleControllerExecutorBuilder { self } + /// Supplies the client config instead of the platform's mock. + /// + /// Needed for Kubernetes, whose controllers have no mockable surface, and for any test that + /// wants to drive a controller against real infrastructure. + pub fn client_config(mut self, config: ClientConfig) -> Self { + self.client_config = Some(config); + self + } + /// Sets a custom cloud client provider. pub fn service_provider(mut self, provider: Arc) -> Self { self.service_provider = Some(provider); @@ -879,8 +890,11 @@ impl SingleControllerExecutorBuilder { }) })?; - // Create platform config with mock values - let client_config = match platform { + // An explicitly supplied config wins, which is what lets a test drive a controller + // against a real cluster instead of a mock. + let client_config = match self.client_config { + Some(config) => config, + None => match platform { Platform::Aws => ClientConfig::Aws(Box::new(AwsClientConfig::mock())), Platform::Gcp => ClientConfig::Gcp(Box::new(GcpClientConfig::mock())), Platform::Azure => ClientConfig::Azure(Box::new(AzureClientConfig::mock())), @@ -888,7 +902,17 @@ impl SingleControllerExecutorBuilder { // Local controllers (e.g. Local Postgres) carry no cloud client; the no-cloud test // config is enough to exercise their platform-agnostic handlers. Platform::Local => ClientConfig::Test, + // Kubernetes has no mock config: every one of its controllers talks to an apiserver, + // so a test must supply a real one via `client_config`. + Platform::Kubernetes => { + return Err(AlienError::new(crate::error::ErrorData::CloudPlatformError { + message: "Platform::Kubernetes needs an explicit client_config — its \ + controllers have no mockable surface".to_string(), + resource_id: None, + })) + } _ => panic!("Unsupported platform for testing: {:?}", platform), + }, }; // Build stack and state directly diff --git a/crates/alien-infra/src/core/registry.rs b/crates/alien-infra/src/core/registry.rs index c73abf299..6f438de5f 100644 --- a/crates/alien-infra/src/core/registry.rs +++ b/crates/alien-infra/src/core/registry.rs @@ -730,6 +730,22 @@ impl ResourceRegistry { >::new()), ); + // Register Local Sandbox controller + #[cfg(feature = "local")] + registry.register_controller_factory( + alien_core::Sandbox::RESOURCE_TYPE, + Platform::Local, + Box::new(DefaultControllerFactory::::new()), + ); + + // Register Kubernetes Sandbox controller + #[cfg(feature = "kubernetes")] + registry.register_controller_factory( + alien_core::Sandbox::RESOURCE_TYPE, + Platform::Kubernetes, + Box::new(DefaultControllerFactory::::new()), + ); + // Register KubernetesCluster controller. The cluster is selected or // created during setup; this runtime controller records substrate // readiness once the agent is installed and reporting. diff --git a/crates/alien-infra/src/core/service_provider.rs b/crates/alien-infra/src/core/service_provider.rs index b5aa428ba..05bc89b1c 100644 --- a/crates/alien-infra/src/core/service_provider.rs +++ b/crates/alien-infra/src/core/service_provider.rs @@ -14,6 +14,7 @@ use alien_aws_clients::{ eventbridge::{EventBridgeApi, EventBridgeClient}, iam::{IamApi, IamClient}, lambda::{LambdaApi, LambdaClient}, + lambda_microvms::{LambdaMicrovmsApi, LambdaMicrovmsClient}, rds::{RdsApi, RdsClient}, s3::{S3Api, S3Client}, secrets_manager::{SecretsManagerApi, SecretsManagerClient}, @@ -25,6 +26,8 @@ use alien_aws_clients::{ use alien_azure_clients::{ application_gateways::{ApplicationGatewayApi, AzureApplicationGatewayClient}, authorization::{AuthorizationApi, AzureAuthorizationClient}, + sandbox_data_plane::{AzureSandboxDataPlaneClient, SandboxDataPlaneApi}, + sandbox_groups::{AzureSandboxGroupsClient, SandboxGroupsApi}, blob_containers::{AzureBlobContainerClient, BlobContainerApi}, cognitive_services::{AzureCognitiveServicesClient, CognitiveServicesAccountsApi}, compute::{AzureVmssClient, VirtualMachineScaleSetsApi}, @@ -75,7 +78,8 @@ use alien_gcp_clients::{ use alien_k8s_clients::{ deployments::DeploymentApi, events::EventApi, jobs::JobApi, kubernetes_client::KubernetesClient, metrics::MetricsApi, nodes::NodeApi, pods::PodApi, - routes::RouteApi, secrets::SecretsApi, services::ServiceApi, version::VersionApi, + routes::RouteApi, runtime_classes::RuntimeClassApi, secrets::SecretsApi, + services::ServiceApi, version::VersionApi, KubernetesClientConfig, }; use std::sync::Arc; @@ -94,6 +98,10 @@ pub trait PlatformServiceProvider: Send + Sync { // AWS clients async fn get_aws_iam_client(&self, config: &AwsClientConfig) -> Result>; async fn get_aws_lambda_client(&self, config: &AwsClientConfig) -> Result>; + async fn get_aws_microvms_client( + &self, + config: &AwsClientConfig, + ) -> Result>; async fn get_aws_s3_client(&self, config: &AwsClientConfig) -> Result>; async fn get_aws_ses_client(&self, config: &AwsClientConfig) -> Result>; async fn get_aws_cloudformation_client( @@ -181,6 +189,16 @@ pub trait PlatformServiceProvider: Send + Sync { &self, config: &AzureClientConfig, ) -> Result>; + fn get_azure_sandbox_groups_client( + &self, + config: &AzureClientConfig, + ) -> Result>; + fn get_azure_sandbox_data_plane_client( + &self, + config: &AzureClientConfig, + region: &str, + resource_group: &str, + ) -> Result>; fn get_azure_blob_container_client( &self, config: &AzureClientConfig, @@ -309,6 +327,11 @@ pub trait PlatformServiceProvider: Send + Sync { config: &'a KubernetesClientConfig, ) -> Result>; #[cfg(feature = "kubernetes")] + async fn get_kubernetes_runtime_class_client<'a>( + &'a self, + config: &'a KubernetesClientConfig, + ) -> Result>; + #[cfg(feature = "kubernetes")] async fn get_kubernetes_metrics_client<'a>( &'a self, config: &'a KubernetesClientConfig, @@ -386,6 +409,13 @@ pub trait PlatformServiceProvider: Send + Sync { None } + #[cfg(feature = "local")] + fn get_local_sandbox_manager(&self) -> Option>; + #[cfg(not(feature = "local"))] + fn get_local_sandbox_manager(&self) -> Option> { + None + } + #[cfg(feature = "local")] fn get_local_queue_manager(&self) -> Option>; #[cfg(not(feature = "local"))] @@ -450,6 +480,22 @@ impl PlatformServiceProvider for DefaultPlatformServiceProvider { ))) } + async fn get_aws_microvms_client( + &self, + config: &AwsClientConfig, + ) -> Result> { + let credentials = AwsCredentialProvider::from_config(config.clone()) + .await + .context(crate::error::ErrorData::CloudPlatformError { + message: "Failed to create AWS credential provider".to_string(), + resource_id: None, + })?; + Ok(Arc::new(LambdaMicrovmsClient::new( + reqwest::Client::new(), + credentials, + ))) + } + async fn get_aws_lambda_client(&self, config: &AwsClientConfig) -> Result> { let credentials = AwsCredentialProvider::from_config(config.clone()) .await @@ -849,6 +895,30 @@ impl PlatformServiceProvider for DefaultPlatformServiceProvider { ))) } + fn get_azure_sandbox_groups_client( + &self, + config: &AzureClientConfig, + ) -> Result> { + Ok(Arc::new(AzureSandboxGroupsClient::new( + reqwest::Client::new(), + AzureTokenCache::new(config.clone()), + ))) + } + + fn get_azure_sandbox_data_plane_client( + &self, + config: &AzureClientConfig, + region: &str, + resource_group: &str, + ) -> Result> { + Ok(Arc::new(AzureSandboxDataPlaneClient::new( + reqwest::Client::new(), + region, + resource_group, + AzureTokenCache::new(config.clone()), + ))) + } + fn get_azure_application_gateway_client( &self, config: &AzureClientConfig, @@ -1113,7 +1183,7 @@ impl PlatformServiceProvider for DefaultPlatformServiceProvider { &'a self, config: &'a KubernetesClientConfig, ) -> Result> { - let client = KubernetesClient::new(config.clone()).await.context( + let client = kubernetes_client(config).await.context( crate::error::ErrorData::CloudPlatformError { message: "Failed to create Kubernetes deployment client".to_string(), resource_id: None, @@ -1127,7 +1197,7 @@ impl PlatformServiceProvider for DefaultPlatformServiceProvider { &'a self, config: &'a KubernetesClientConfig, ) -> Result> { - let client = KubernetesClient::new(config.clone()).await.context( + let client = kubernetes_client(config).await.context( crate::error::ErrorData::CloudPlatformError { message: "Failed to create Kubernetes job client".to_string(), resource_id: None, @@ -1141,7 +1211,7 @@ impl PlatformServiceProvider for DefaultPlatformServiceProvider { &'a self, config: &'a KubernetesClientConfig, ) -> Result> { - let client = KubernetesClient::new(config.clone()).await.context( + let client = kubernetes_client(config).await.context( crate::error::ErrorData::CloudPlatformError { message: "Failed to create Kubernetes pod client".to_string(), resource_id: None, @@ -1155,7 +1225,7 @@ impl PlatformServiceProvider for DefaultPlatformServiceProvider { &'a self, config: &'a KubernetesClientConfig, ) -> Result> { - let client = KubernetesClient::new(config.clone()).await.context( + let client = kubernetes_client(config).await.context( crate::error::ErrorData::CloudPlatformError { message: "Failed to create Kubernetes event client".to_string(), resource_id: None, @@ -1164,12 +1234,26 @@ impl PlatformServiceProvider for DefaultPlatformServiceProvider { Ok(Arc::new(client)) } + #[cfg(feature = "kubernetes")] + async fn get_kubernetes_runtime_class_client<'a>( + &'a self, + config: &'a KubernetesClientConfig, + ) -> Result> { + let client = kubernetes_client(config).await.context( + crate::error::ErrorData::CloudPlatformError { + message: "Failed to create Kubernetes runtime class client".to_string(), + resource_id: None, + }, + )?; + Ok(Arc::new(client)) + } + #[cfg(feature = "kubernetes")] async fn get_kubernetes_node_client<'a>( &'a self, config: &'a KubernetesClientConfig, ) -> Result> { - let client = KubernetesClient::new(config.clone()).await.context( + let client = kubernetes_client(config).await.context( crate::error::ErrorData::CloudPlatformError { message: "Failed to create Kubernetes node client".to_string(), resource_id: None, @@ -1183,7 +1267,7 @@ impl PlatformServiceProvider for DefaultPlatformServiceProvider { &'a self, config: &'a KubernetesClientConfig, ) -> Result> { - let client = KubernetesClient::new(config.clone()).await.context( + let client = kubernetes_client(config).await.context( crate::error::ErrorData::CloudPlatformError { message: "Failed to create Kubernetes metrics client".to_string(), resource_id: None, @@ -1197,7 +1281,7 @@ impl PlatformServiceProvider for DefaultPlatformServiceProvider { &'a self, config: &'a KubernetesClientConfig, ) -> Result> { - let client = KubernetesClient::new(config.clone()).await.context( + let client = kubernetes_client(config).await.context( crate::error::ErrorData::CloudPlatformError { message: "Failed to create Kubernetes secrets client".to_string(), resource_id: None, @@ -1211,7 +1295,7 @@ impl PlatformServiceProvider for DefaultPlatformServiceProvider { &'a self, config: &'a KubernetesClientConfig, ) -> Result> { - let client = KubernetesClient::new(config.clone()).await.context( + let client = kubernetes_client(config).await.context( crate::error::ErrorData::CloudPlatformError { message: "Failed to create Kubernetes service client".to_string(), resource_id: None, @@ -1225,7 +1309,7 @@ impl PlatformServiceProvider for DefaultPlatformServiceProvider { &'a self, config: &'a KubernetesClientConfig, ) -> Result> { - let client = KubernetesClient::new(config.clone()).await.context( + let client = kubernetes_client(config).await.context( crate::error::ErrorData::CloudPlatformError { message: "Failed to create Kubernetes route client".to_string(), resource_id: None, @@ -1239,7 +1323,7 @@ impl PlatformServiceProvider for DefaultPlatformServiceProvider { &'a self, config: &'a KubernetesClientConfig, ) -> Result> { - let client = KubernetesClient::new(config.clone()).await.context( + let client = kubernetes_client(config).await.context( crate::error::ErrorData::CloudPlatformError { message: "Failed to create Kubernetes version client".to_string(), resource_id: None, @@ -1296,6 +1380,11 @@ impl PlatformServiceProvider for DefaultPlatformServiceProvider { .and_then(|p| p.container_manager()) } + #[cfg(feature = "local")] + fn get_local_sandbox_manager(&self) -> Option> { + self.local_bindings.as_ref().and_then(|p| p.sandbox_manager()) + } + #[cfg(feature = "local")] fn get_local_queue_manager(&self) -> Option> { self.local_bindings @@ -1308,3 +1397,16 @@ impl PlatformServiceProvider for DefaultPlatformServiceProvider { self.local_bindings.clone() } } + +/// Builds a Kubernetes client, resolving a kubeconfig reference first. +/// +/// `KubernetesClientConfig::Kubeconfig` is a placeholder the client refuses, and it names this +/// crate as where the resolution belongs. Routing every client through here is what makes that +/// true, rather than each caller resolving for itself or, as before, nobody doing it. +#[cfg(feature = "kubernetes")] +async fn kubernetes_client( + config: &KubernetesClientConfig, +) -> alien_client_core::Result { + let resolved = crate::kubeconfig::resolve_kubeconfig(config).await?; + KubernetesClient::new(resolved).await +} diff --git a/crates/alien-infra/src/kubeconfig.rs b/crates/alien-infra/src/kubeconfig.rs index 82c82231a..80a590884 100644 --- a/crates/alien-infra/src/kubeconfig.rs +++ b/crates/alien-infra/src/kubeconfig.rs @@ -753,6 +753,67 @@ fn parse_certificates(data: &[u8]) -> Result>> { .collect()) } +/// Turns a kubeconfig reference into the concrete config the Kubernetes client accepts. +/// +/// The client refuses `Kubeconfig` outright and says so: resolution belongs here, because +/// reading the file means reading CA and client-certificate paths off disk and running an +/// exec-plugin credential command, none of which a cloud client should do. Any other variant +/// passes through, so this is safe to call unconditionally on whatever config arrives. +pub async fn resolve_kubeconfig( + config: &alien_k8s_clients::KubernetesClientConfig, +) -> Result { + use alien_k8s_clients::KubernetesClientConfig; + + let KubernetesClientConfig::Kubeconfig { + kubeconfig_path, + context, + cluster, + user, + namespace, + additional_headers, + } = config + else { + return Ok(config.clone()); + }; + + let file = match kubeconfig_path { + Some(path) => Kubeconfig::read_from(path.clone())?, + None => Kubeconfig::read()?, + }; + + let loader = ConfigLoader::load(file, context.as_ref(), cluster.as_ref(), user.as_ref())?; + + let server_url = loader.cluster.server.clone().ok_or_else(|| { + AlienError::new(ErrorData::KubeconfigError { + message: "the selected cluster names no server".to_string(), + }) + })?; + + Ok(KubernetesClientConfig::Manual { + server_url, + certificate_authority_data: loader + .cluster + .load_certificate_authority()? + .map(|bytes| general_purpose::STANDARD.encode(bytes)), + insecure_skip_tls_verify: loader.cluster.insecure_skip_tls_verify, + client_certificate_data: loader + .user + .load_client_certificate()? + .map(|bytes| general_purpose::STANDARD.encode(bytes)), + client_key_data: loader + .user + .load_client_key()? + .map(|bytes| general_purpose::STANDARD.encode(bytes)), + token: loader.user.load_token().await?, + username: loader.user.username.clone(), + password: loader.user.password.clone(), + namespace: namespace + .clone() + .or_else(|| loader.current_context.namespace.clone()), + additional_headers: additional_headers.clone().unwrap_or_default(), + }) +} + #[cfg(test)] mod tests { use super::*; @@ -760,6 +821,99 @@ mod tests { use std::io::Write; use tempfile::NamedTempFile; + /// The client refuses `Kubeconfig` and names this crate as where resolution belongs, so the + /// resolved config has to carry everything the client needs: server, CA, and a credential. + #[tokio::test] + async fn a_kubeconfig_resolves_into_a_manual_config() { + let mut file = NamedTempFile::new().expect("temp file"); + write!( + file, + r#" +apiVersion: v1 +kind: Config +current-context: ctx +clusters: +- cluster: + certificate-authority-data: dGVzdA== + server: https://cluster.example:6443 + name: c +contexts: +- context: + cluster: c + user: u + namespace: from-context + name: ctx +users: +- name: u + user: + token: inline-token +"# + ) + .expect("write kubeconfig"); + + let resolved = resolve_kubeconfig(&KubernetesClientConfig::Kubeconfig { + kubeconfig_path: Some(file.path().display().to_string()), + context: None, + cluster: None, + user: None, + namespace: None, + additional_headers: None, + }) + .await + .expect("a kubeconfig with an inline token resolves"); + + let KubernetesClientConfig::Manual { + server_url, + certificate_authority_data, + token, + namespace, + .. + } = resolved + else { + panic!("resolution must produce a Manual config"); + }; + + assert_eq!(server_url, "https://cluster.example:6443"); + assert_eq!(certificate_authority_data.as_deref(), Some("dGVzdA==")); + assert_eq!(token.as_deref(), Some("inline-token")); + assert_eq!( + namespace.as_deref(), + Some("from-context"), + "the context's namespace is the default when the caller names none" + ); + } + + /// Safe to call on whatever config arrives, so callers do not have to match first. + #[tokio::test] + async fn every_other_variant_passes_through_untouched() { + let manual = KubernetesClientConfig::Manual { + server_url: "https://test:6443".to_string(), + certificate_authority_data: None, + insecure_skip_tls_verify: None, + client_certificate_data: None, + client_key_data: None, + token: Some("t".to_string()), + username: None, + password: None, + namespace: None, + additional_headers: HashMap::new(), + }; + + assert_eq!( + resolve_kubeconfig(&manual).await.expect("passes through"), + manual + ); + + let in_cluster = KubernetesClientConfig::InCluster { + additional_headers: None, + namespace: Some("default".to_string()), + }; + assert_eq!( + resolve_kubeconfig(&in_cluster).await.expect("passes through"), + in_cluster + ); + } + #[test] fn test_exec_config_parsing() { let kubeconfig_content = r#" diff --git a/crates/alien-infra/src/lib.rs b/crates/alien-infra/src/lib.rs index 31f208bd3..bfe607b07 100644 --- a/crates/alien-infra/src/lib.rs +++ b/crates/alien-infra/src/lib.rs @@ -44,6 +44,10 @@ mod compute_cluster; #[cfg(feature = "local")] pub use compute_cluster::*; +mod sandbox; +#[cfg(feature = "local")] +pub use sandbox::*; + mod kubernetes_cluster; #[cfg(feature = "kubernetes")] diff --git a/crates/alien-infra/src/network/aws_import.rs b/crates/alien-infra/src/network/aws_import.rs index a4fa434a3..d74af9927 100644 --- a/crates/alien-infra/src/network/aws_import.rs +++ b/crates/alien-infra/src/network/aws_import.rs @@ -276,4 +276,165 @@ mod tests { .expect("imported network should have controller state"); assert_eq!(internal["isByoVpc"], true); } + + /// The `use-default` import hands off half-finished on purpose: status + /// `Provisioning`, state `CreateStart`, no VPC id, leaving the controller to + /// discover the account default VPC. Nothing covered that the two halves + /// compose, so this pins that stepping the imported state actually reaches + /// discovery. If it ever stops, every resource depending on the network + /// stalls behind it and initial setup never completes. + #[tokio::test] + async fn imported_use_default_network_reaches_default_vpc_discovery() { + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::Arc; + + use alien_aws_clients::ec2::MockEc2Api; + use alien_core::{Platform, ResourceStatus}; + + use crate::core::{controller_test::SingleControllerExecutor, MockPlatformServiceProvider}; + + let settings = StackSettings { + network: Some(NetworkSettings::UseDefault), + ..StackSettings::default() + }; + let entry = network_entry(); + let imported = AwsNetworkImporter + .import( + empty_default_import_data(), + &import_context(&settings, &entry), + ) + .expect("network import should succeed"); + assert_eq!(imported.status, ResourceStatus::Provisioning); + + let controller: AwsNetworkController = + serde_json::from_value(imported.internal_state.expect("controller state")) + .expect("controller should deserialize from its imported state"); + + let discovered = Arc::new(AtomicBool::new(false)); + let flag = discovered.clone(); + let mut ec2 = MockEc2Api::new(); + ec2.expect_describe_vpcs().returning(move |_| { + flag.store(true, Ordering::SeqCst); + Err(alien_error::AlienError::new( + alien_client_core::ErrorData::RemoteServiceUnavailable { + message: "stop after the lookup; provisioning itself is covered elsewhere" + .to_string(), + }, + )) + }); + let ec2 = Arc::new(ec2); + let mut provider = MockPlatformServiceProvider::new(); + provider + .expect_get_aws_ec2_client() + .returning(move |_| Ok(ec2.clone())); + + let network = Network::new("default-network".to_string()) + .settings(NetworkSettings::UseDefault) + .build(); + let mut executor = SingleControllerExecutor::builder() + .resource(network) + .controller(controller) + .platform(Platform::Aws) + .stack_settings(settings) + .service_provider(Arc::new(provider)) + .build() + .await + .expect("executor should build"); + + let _ = executor.step().await; + assert!( + discovered.load(Ordering::SeqCst), + "stepping the imported use-default network must reach default-VPC discovery" + ); + } + + /// The production initial-setup path, not the controller in isolation: + /// `continue_imported` with the Frozen filter, exactly as `initial_setup.rs` + /// builds it. A `use-default` network arrives here unfinished by design, so + /// this run is the only thing that can finish it. + #[tokio::test] + async fn initial_setup_drives_the_imported_use_default_network() { + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::Arc; + + use alien_aws_clients::ec2::MockEc2Api; + use alien_aws_clients::{AwsClientConfig, AwsClientConfigExt as _}; + use alien_core::{ + ClientConfig, DeploymentConfig, EnvironmentVariablesSnapshot, ExternalBindings, + InitialSetupAuthority, Platform, Stack, StackState, + }; + + use crate::core::{MockPlatformServiceProvider, StackExecutor}; + + let settings = StackSettings { + network: Some(NetworkSettings::UseDefault), + ..StackSettings::default() + }; + let entry = network_entry(); + let imported = AwsNetworkImporter + .import( + empty_default_import_data(), + &import_context(&settings, &entry), + ) + .expect("network import should succeed"); + + let network = Network::new("default-network".to_string()) + .settings(NetworkSettings::UseDefault) + .build(); + let stack = Stack::new("use-default-handoff".to_string()) + .add(network, ResourceLifecycle::Frozen) + .build(); + let mut state = StackState::new(Platform::Aws); + state + .resources + .insert("default-network".to_string(), imported); + + let discovered = Arc::new(AtomicBool::new(false)); + let flag = discovered.clone(); + let mut ec2 = MockEc2Api::new(); + ec2.expect_describe_vpcs().returning(move |_| { + flag.store(true, Ordering::SeqCst); + Err(alien_error::AlienError::new( + alien_client_core::ErrorData::RemoteServiceUnavailable { + message: "stop after the lookup".to_string(), + }, + )) + }); + let ec2 = Arc::new(ec2); + let mut provider = MockPlatformServiceProvider::new(); + provider + .expect_get_aws_ec2_client() + .returning(move |_| Ok(ec2.clone())); + + let config = DeploymentConfig::builder() + .stack_settings(settings) + .environment_variables(EnvironmentVariablesSnapshot { + variables: vec![], + hash: String::new(), + created_at: String::new(), + }) + .external_bindings(ExternalBindings::default()) + .allow_frozen_changes(false) + .build(); + + let executor = StackExecutor::builder( + &stack, + ClientConfig::Aws(Box::new(AwsClientConfig::mock())), + ) + .deployment_config(&config) + .service_provider(Arc::new(provider)) + .initial_setup_authority(InitialSetupAuthority::ImportedHandoff) + .lifecycle_filter(vec![ResourceLifecycle::Frozen]) + .step_running_resources(false) + .build() + .expect("executor should build"); + + let _ = executor.continue_imported(state).await; + assert!( + discovered.load(Ordering::SeqCst), + "initial setup must drive the imported use-default network to default-VPC \ + discovery; without it the network stays Provisioning forever and every \ + resource depending on it stalls behind it" + ); + } } diff --git a/crates/alien-infra/src/sandbox/kubernetes.rs b/crates/alien-infra/src/sandbox/kubernetes.rs new file mode 100644 index 000000000..23338d84a --- /dev/null +++ b/crates/alien-infra/src/sandbox/kubernetes.rs @@ -0,0 +1,603 @@ +//! Kubernetes Sandbox controller. +//! +//! Alien owns the lifecycle here, unlike Postgres or KV on Kubernetes which connect to +//! something the operator already runs. So this is a real controller rather than an +//! external-binding shim. +//! +//! The Frozen parent — namespace, ServiceAccount, NetworkPolicy, pod template — is emitted by +//! Helm at setup. What the controller owns is refusing an ineligible cluster before anything is +//! created, and reaping the Live pods. + +use std::time::Duration; + +use tracing::{debug, info, warn}; + +use crate::core::ResourceControllerContext; +use crate::error::{ErrorData, Result}; +use crate::sandbox::{ + idle_pool_pod, idle_selector, pool_deficit, require_sandboxed_runtime_class, LABEL_SANDBOX, +}; +use alien_core::{ResourceOutputs as CoreResourceOutputs, ResourceStatus, Sandbox, SandboxOutputs}; +use alien_error::{AlienError, Context, IntoAlienError}; +use alien_macros::controller; + +/// Runtime class a sandbox pod runs under when the operator has not chosen one. +const DEFAULT_RUNTIME_CLASS: &str = "gvisor"; + +/// Kubernetes Sandbox controller. +#[controller] +pub struct KubernetesSandboxController { + /// Sandbox this controller owns, and the enumeration scope for its pods. + pub(crate) sandbox_id: Option, + /// Namespace the Helm chart created for this sandbox's pods. + pub(crate) namespace: Option, + /// Runtime class every session pod carries. + pub(crate) runtime_class: Option, + /// Idle pods kept ready. 79s cold against 2.7s warm is the reason this exists. + pub(crate) warm_pool_size: Option, + /// Public half of the sandbox's capability keypair, base64. Only the public half: a pod's + /// environment is readable by the untrusted code inside it. + pub(crate) capability_public_key: Option, + /// Where the application reaches the session broker. Set from the operator's own service + /// address, because the broker is served by the operator. + pub(crate) broker_url: Option, +} + +#[controller] +impl KubernetesSandboxController { + // ─────────────── CREATE FLOW ─────────────────────────────────────────── + + #[flow_entry(Create)] + #[handler( + state = VerifyCluster, + on_failure = ProvisionFailed, + status = ResourceStatus::Provisioning + )] + async fn verify_cluster( + &mut self, + ctx: &ResourceControllerContext<'_>, + ) -> Result { + let config = ctx.desired_resource_config::()?; + let runtime_class = self + .runtime_class + .clone() + .unwrap_or_else(|| DEFAULT_RUNTIME_CLASS.to_string()); + + let kubernetes_config = ctx.get_kubernetes_config()?; + let client = ctx + .service_provider + .get_kubernetes_runtime_class_client(kubernetes_config) + .await?; + + let available = client + .list_runtime_classes() + .await + .context(ErrorData::CloudPlatformError { + message: "Failed to list RuntimeClasses".to_string(), + resource_id: Some(config.id.clone()), + })?; + + // Before any pod exists. On Autopilot an unschedulable pod is not rejected — node + // auto-provisioning picks it up and it sits in Pending while nodes are billed. + require_sandboxed_runtime_class(&config.id, &runtime_class, &available.items)?; + + self.runtime_class = Some(runtime_class); + self.sandbox_id = Some(config.id.clone()); + self.broker_url = broker_url(); + + // Recorded as soon as it is known, because the binding and the outputs both read it + // directly rather than through the fallback: leaving it unset publishes no binding at + // all, and the sandbox comes up Running with nothing able to reach it. + let namespace = deployment_namespace(ctx.get_kubernetes_config()?)?; + self.namespace = Some(namespace.clone()); + + if self.capability_public_key.is_none() { + self.capability_public_key = Some(ensure_capability_keypair(ctx, &config.id, &namespace).await?); + } + + info!(sandbox_id = %config.id, "Cluster can run sandboxes"); + + Ok(HandlerAction::Continue { + state: Ready, + suggested_delay: None, + }) + } + + #[handler( + state = Ready, + on_failure = RefreshFailed, + status = ResourceStatus::Running + )] + async fn ready(&mut self, ctx: &ResourceControllerContext<'_>) -> Result { + let config = ctx.desired_resource_config::()?; + + let namespace = deployment_namespace(ctx.get_kubernetes_config()?)?; + let sessions = list_session_pods(ctx, &namespace, &config.id).await?; + + let created = replenish_warm_pool( + ctx, + &config, + &namespace, + self.runtime_class.as_deref().unwrap_or(DEFAULT_RUNTIME_CLASS), + self.warm_pool_size.unwrap_or(DEFAULT_WARM_POOL_SIZE), + self.capability_public_key.as_deref(), + ) + .await?; + + debug!(sandbox_id = %config.id, sessions, created, "Sandbox health check passed"); + + Ok(HandlerAction::Continue { + state: Ready, + suggested_delay: Some(Duration::from_secs(30)), + }) + } + + // ─────────────── UPDATE FLOW ────────────────────────────────────────── + + #[flow_entry(Update, from = [Ready, RefreshFailed])] + #[handler( + state = UpdatingSandbox, + on_failure = UpdateFailed, + status = ResourceStatus::Updating + )] + async fn updating_sandbox( + &mut self, + ctx: &ResourceControllerContext<'_>, + ) -> Result { + let config = ctx.desired_resource_config::()?; + + // Config applies to pods created after it. Running sessions are not rolled: a session + // is a unit of work someone is waiting on, not a replica. + info!(sandbox_id = %config.id, "Updated Kubernetes sandbox configuration"); + + Ok(HandlerAction::Continue { + state: Ready, + suggested_delay: None, + }) + } + + // ─────────────── DELETE FLOW ────────────────────────────────────────── + + #[flow_entry(Delete)] + #[handler( + state = Deleting, + on_failure = DeleteFailed, + status = ResourceStatus::Deleting + )] + async fn deleting(&mut self, ctx: &ResourceControllerContext<'_>) -> Result { + let config = ctx.desired_resource_config::()?; + + let namespace = deployment_namespace(ctx.get_kubernetes_config()?)?; + let kubernetes_config = ctx.get_kubernetes_config()?; + let client = ctx + .service_provider + .get_kubernetes_pod_client(kubernetes_config) + .await?; + + let pods = client + .list_pods( + &namespace, + Some(format!("{LABEL_SANDBOX}={}", config.id)), + None, + ) + .await + .context(ErrorData::CloudPlatformError { + message: "Failed to list sandbox pods for deletion".to_string(), + resource_id: Some(config.id.clone()), + })?; + + // Children before the parent, and best-effort per pod: one already-gone pod must not + // strand the rest. But only *already-gone* — a delete refused for any other reason + // (forbidden, a webhook, the apiserver unavailable) leaves a sandbox running, and + // reporting Deleted over it is a teardown that lies. + let mut removed = 0; + let mut unreachable = Vec::new(); + for pod in &pods.items { + let Some(name) = pod.metadata.name.as_deref() else { + continue; + }; + + match client.delete_pod(&namespace, name).await { + Ok(()) => removed += 1, + Err(error) if error.code == "REMOTE_RESOURCE_NOT_FOUND" => removed += 1, + Err(error) => unreachable.push(format!("{name}: {error}")), + } + } + + if !unreachable.is_empty() { + return Err(AlienError::new(ErrorData::CloudPlatformError { + message: format!( + "{} sandbox pod(s) could not be deleted and are still running: {}", + unreachable.len(), + unreachable.join("; ") + ), + resource_id: Some(config.id.clone()), + })); + } + + // The capability key outlives its pods otherwise, and a signing key nobody can use is + // still a signing key sitting in the cluster. Best effort: one already gone is the + // desired end state, and it must not strand the rest of the teardown. + let secrets = ctx + .service_provider + .get_kubernetes_secrets_client(ctx.get_kubernetes_config()?) + .await?; + let _ = secrets + .delete_secret(&namespace, &capability_secret_name(&config.id)) + .await; + + info!(sandbox_id = %config.id, removed, "Removed Kubernetes sandbox pods and capability key"); + + Ok(HandlerAction::Continue { + state: Deleted, + suggested_delay: None, + }) + } + + fn get_binding_params(&self) -> Result> { + use alien_core::bindings::{BindingValue, SandboxBinding}; + + // Nothing to publish until the broker has an address and the pool has a key: a binding + // pointing at neither would fail on first use rather than fail to appear. + let (Some(sandbox_id), Some(namespace), Some(runtime_class), Some(broker_url)) = ( + self.sandbox_id.clone(), + self.namespace.clone(), + self.runtime_class.clone(), + self.broker_url.clone(), + ) else { + return Ok(None); + }; + + let binding = SandboxBinding::kubernetes( + BindingValue::value(namespace), + BindingValue::value(runtime_class), + BindingValue::value(idle_selector(&sandbox_id)), + BindingValue::value(broker_url), + BindingValue::value(capability_secret_name(&sandbox_id)), + BindingValue::value(SERVICE_ACCOUNT_TOKEN_PATH.to_string()), + ); + + Ok(Some(serde_json::to_value(binding).into_alien_error().context( + ErrorData::ResourceStateSerializationFailed { + resource_id: "binding".to_string(), + message: "Failed to serialize the sandbox binding".to_string(), + }, + )?)) + } + + // ─────────────── TERMINAL STATES ────────────────────────────────────── + + terminal_state!(state = Deleted, status = ResourceStatus::Deleted); + terminal_state!( + state = ProvisionFailed, + status = ResourceStatus::ProvisionFailed + ); + terminal_state!(state = UpdateFailed, status = ResourceStatus::UpdateFailed); + terminal_state!(state = DeleteFailed, status = ResourceStatus::DeleteFailed); + terminal_state!( + state = RefreshFailed, + status = ResourceStatus::RefreshFailed + ); + + // ─────────────── HELPER METHODS ────────────────────────────────────── + + fn build_outputs(&self) -> Option { + self.namespace.as_ref().map(|namespace| { + CoreResourceOutputs::new(SandboxOutputs { + parent_name: namespace.clone(), + identifier: self.runtime_class.clone(), + // Sessions are reached through the in-pod agent; there is no provider endpoint. + endpoint: None, + }) + }) + } +} + +/// Idle pods kept ready when the operator has not chosen a size. +const DEFAULT_WARM_POOL_SIZE: usize = 2; + +/// Tops the idle pool back up to its target. +/// +/// Runs on the health tick rather than on session create: a create that had to wait for a pod +/// to be built would be the 79s cold path this pool exists to avoid. +async fn replenish_warm_pool( + ctx: &ResourceControllerContext<'_>, + sandbox: &Sandbox, + namespace: &str, + runtime_class: &str, + target: usize, + capability_public_key: Option<&str>, +) -> Result { + let kubernetes_config = ctx.get_kubernetes_config()?; + let client = ctx + .service_provider + .get_kubernetes_pod_client(kubernetes_config) + .await?; + + let idle = client + .list_pods(namespace, Some(idle_selector(&sandbox.id)), None) + .await + .context(ErrorData::CloudPlatformError { + message: "Failed to list idle sandbox pods".to_string(), + resource_id: Some(sandbox.id.clone()), + })?; + + let deficit = pool_deficit(target, idle.items.len()); + let mut created = 0; + + for _ in 0..deficit { + let pod = idle_pool_pod( + sandbox, + namespace, + runtime_class, + None, + capability_public_key, + ); + + // Best effort per pod: a pool that is one short is slower, not broken, and failing the + // health tick over it would take a working sandbox out of Running. The reason is logged + // rather than swallowed, because a pool that is always empty is a bug and silence makes + // it look like a slow cluster. + match client.create_pod(namespace, &pod).await { + Ok(_) => created += 1, + Err(error) => { + warn!(sandbox_id = %sandbox.id, %namespace, error = %error, "Failed to create a pool pod") + } + } + } + + Ok(created) +} + +/// The deployment's namespace, which is where a sandbox's pods go. +/// +/// Not a namespace of the sandbox's own: the operator is namespace-scoped and holds no +/// cluster-admin, so it can neither create a namespace nor act in one it was not installed into. +/// Every resource lives in the one namespace Helm created, and a sandbox's pods are separated +/// from everything else there by the `alien.dev/sandbox` label their NetworkPolicy selects on. +/// +/// This is also the namespace the broker checks a caller's ServiceAccount against, so reading it +/// from anywhere else would put the two halves in different places. +fn deployment_namespace(config: &alien_core::KubernetesClientConfig) -> Result { + use alien_core::KubernetesClientConfig as Config; + + let namespace = match config { + Config::InCluster { namespace, .. } | Config::Kubeconfig { namespace, .. } => { + namespace.clone() + } + _ => None, + }; + + namespace.ok_or_else(|| { + AlienError::new(ErrorData::CloudPlatformError { + message: "the Kubernetes client config names no namespace, so there is nowhere to \ + put a sandbox's pods" + .to_string(), + resource_id: None, + }) + }) +} + +/// Counts the Live pods belonging to one sandbox. +async fn list_session_pods( + ctx: &ResourceControllerContext<'_>, + namespace: &str, + sandbox_id: &str, +) -> Result { + let kubernetes_config = ctx.get_kubernetes_config()?; + let client = ctx + .service_provider + .get_kubernetes_pod_client(kubernetes_config) + .await?; + + let pods = client + .list_pods(namespace, Some(format!("{LABEL_SANDBOX}={sandbox_id}")), None) + .await + .context(ErrorData::CloudPlatformError { + message: "Failed to list sandbox pods".to_string(), + resource_id: Some(sandbox_id.to_string()), + })?; + + Ok(pods.items.len()) +} + +#[cfg(test)] +mod tests { + /// A binding pointing at no broker would fail on first use rather than fail to appear, so + /// nothing is published until the address and the key are both known. + #[test] + fn no_binding_is_published_before_the_broker_has_an_address() { + use crate::core::ResourceController; + + let mut controller = KubernetesSandboxController { + state: KubernetesSandboxState::Ready, + sandbox_id: Some("sbx".to_string()), + namespace: Some("alien-sandbox-sbx".to_string()), + runtime_class: Some("gvisor".to_string()), + warm_pool_size: None, + capability_public_key: Some("cHVibGlj".to_string()), + broker_url: None, + _internal_stay_count: None, + }; + + assert!(controller.get_binding_params().expect("params").is_none()); + + controller.broker_url = Some("http://alien-operator.alien:8080".to_string()); + let params = controller + .get_binding_params() + .expect("params") + .expect("a ready sandbox publishes a binding"); + + assert_eq!(params["brokerUrl"], "http://alien-operator.alien:8080"); + assert_eq!(params["keyName"], "alien-sandbox-sbx-capability"); + assert_eq!( + params["tokenPath"], + "/var/run/secrets/kubernetes.io/serviceaccount/token" + ); + // A path Kubernetes already wrote, never a secret of ours. + assert!( + !params.to_string().contains("cHVibGlj"), + "no key material belongs in a binding: {params}" + ); + } + + use super::*; + use crate::core::{deserialize_controller, serialize_controller, ResourceController}; + + #[test] + fn controller_round_trips_by_tag() { + let controller = KubernetesSandboxController { + namespace: Some("alien-sandbox-agent".to_string()), + runtime_class: Some("gvisor".to_string()), + ..Default::default() + }; + + let value = serialize_controller(&controller).expect("serializes with its tag"); + assert_eq!(value["type"], "KubernetesSandboxController"); + + let restored = deserialize_controller(value).expect("a registered tag must deserialize"); + assert_eq!(restored.controller_type(), controller.controller_type()); + } + + #[test] + fn the_registry_resolves_a_kubernetes_sandbox_controller() { + let registry = crate::core::ResourceRegistry::with_built_ins(); + + let controller = registry + .get_controller( + alien_core::Sandbox::RESOURCE_TYPE, + alien_core::Platform::Kubernetes, + ) + .expect("Kubernetes must have a registered Sandbox controller"); + assert_eq!(controller.controller_type(), "KubernetesSandboxController"); + } +} + +/// Name of the Secret holding a sandbox's capability signing key. +pub fn capability_secret_name(sandbox_id: &str) -> String { + format!("alien-sandbox-{sandbox_id}-capability") +} + +/// Key within that Secret. +const CAPABILITY_SECRET_KEY: &str = "signingKey"; + +/// Creates the sandbox's capability keypair if it has none, returning the public half. +/// +/// The private half goes into a Kubernetes Secret that sandbox pods never mount: the broker +/// reads it to mint, the agent only ever sees the public half. Storing it in controller state +/// would put a signing key in deployment state, which is the one thing state must not carry. +/// +/// Reads before it writes, so a controller restart adopts the existing key rather than minting +/// a second one that would invalidate every capability already handed out. +#[cfg(feature = "kubernetes")] +async fn ensure_capability_keypair( + ctx: &ResourceControllerContext<'_>, + sandbox_id: &str, + namespace: &str, +) -> Result { + use base64::engine::general_purpose::STANDARD as BASE64; + use base64::Engine as _; + use k8s_openapi::api::core::v1::Secret; + use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta; + + let kubernetes_config = ctx.get_kubernetes_config()?; + let client = ctx + .service_provider + .get_kubernetes_secrets_client(kubernetes_config) + .await?; + + let name = capability_secret_name(sandbox_id); + + if let Ok(existing) = client.get_secret(namespace, &name).await { + if let Some(encoded) = existing + .data + .as_ref() + .and_then(|data| data.get(CAPABILITY_SECRET_KEY)) + { + let pair = ed25519_compact::KeyPair::from_slice(&encoded.0).map_err(|error| { + AlienError::new(ErrorData::CloudPlatformError { + message: format!("the stored capability key is unusable: {error}"), + resource_id: Some(sandbox_id.to_string()), + }) + })?; + return Ok(BASE64.encode(pair.pk.as_ref())); + } + } + + let pair = ed25519_compact::KeyPair::generate(); + + let secret = Secret { + metadata: ObjectMeta { + name: Some(name.clone()), + namespace: Some(namespace.to_string()), + labels: Some(std::collections::BTreeMap::from([( + LABEL_SANDBOX.to_string(), + sandbox_id.to_string(), + )])), + ..Default::default() + }, + data: Some(std::collections::BTreeMap::from([( + CAPABILITY_SECRET_KEY.to_string(), + k8s_openapi::ByteString(pair.as_ref().to_vec()), + )])), + ..Default::default() + }; + + if client.create_secret(namespace, &secret).await.is_err() { + // Losing this write means somebody else created the key first, which is the outcome the + // read above is for — adopt theirs. Propagating instead would put a sandbox whose key is + // live and usable into a terminal ProvisionFailed, for a race that already resolved. + let existing = client + .get_secret(namespace, &name) + .await + .context(ErrorData::CloudPlatformError { + message: format!("failed to store the capability key for '{sandbox_id}'"), + resource_id: Some(sandbox_id.to_string()), + })?; + + let encoded = existing + .data + .as_ref() + .and_then(|data| data.get(CAPABILITY_SECRET_KEY)) + .ok_or_else(|| { + AlienError::new(ErrorData::CloudPlatformError { + message: format!("the capability Secret for '{sandbox_id}' carries no key"), + resource_id: Some(sandbox_id.to_string()), + }) + })?; + + let adopted = ed25519_compact::KeyPair::from_slice(&encoded.0).map_err(|error| { + AlienError::new(ErrorData::CloudPlatformError { + message: format!("the stored capability key is unusable: {error}"), + resource_id: Some(sandbox_id.to_string()), + }) + })?; + return Ok(BASE64.encode(adopted.pk.as_ref())); + } + + Ok(BASE64.encode(pair.pk.as_ref())) +} + +/// Where Kubernetes mounts a pod's own ServiceAccount token. +/// +/// The binding carries this path rather than any secret of ours: the platform put the file +/// there, and the broker verifies it with a `TokenReview`. +const SERVICE_ACCOUNT_TOKEN_PATH: &str = "/var/run/secrets/kubernetes.io/serviceaccount/token"; + +/// Where the application reaches the session broker. +/// +/// Derived from the operator's own identity rather than configured: the broker is served by the +/// operator, and the Helm chart already gives it a Service named after `OPERATOR_NAME` in +/// `KUBERNETES_NAMESPACE`. A separate setting would be a second place for the same fact to be +/// wrong. +/// +/// `None` when the operator is not running in a cluster, which is also when there is no Service +/// to address and therefore nothing to publish. +#[cfg(feature = "kubernetes")] +fn broker_url() -> Option { + let name = std::env::var("OPERATOR_NAME").ok()?; + let namespace = std::env::var("KUBERNETES_NAMESPACE").ok()?; + let port = std::env::var("OTLP_PORT").unwrap_or_else(|_| "8080".to_string()); + + Some(format!("http://{name}.{namespace}.svc:{port}")) +} diff --git a/crates/alien-infra/src/sandbox/kubernetes_broker.rs b/crates/alien-infra/src/sandbox/kubernetes_broker.rs new file mode 100644 index 000000000..0072badb3 --- /dev/null +++ b/crates/alien-infra/src/sandbox/kubernetes_broker.rs @@ -0,0 +1,561 @@ +//! Claiming a warm pod and minting the capability that reaches its agent. +//! +//! This is the whole reason a broker exists. Claiming is a `PATCH` on pods, and putting that in +//! the binding would mean the customer's application holds pod-write on the namespace. That is +//! too much: `pods/exec` reaches every pod in the namespace, and the Docker socket is +//! root-equivalent on the host. A narrower credential is worth a process. +//! +//! The application gets back a pod address and a capability scoped to one session. It never gets +//! a cluster credential. + +use std::sync::Arc; + +use base64::engine::general_purpose::STANDARD as BASE64; +use base64::Engine as _; +use k8s_openapi::api::core::v1::Pod; + +use crate::error::{ErrorData, Result}; +use crate::sandbox::{claim_idle_pod, idle_selector}; +use alien_core::sandbox_capability::{SandboxCapabilityClaims, SandboxOperationClass}; +use alien_core::sandbox_capability_token; +use alien_error::{AlienError, Context}; +use alien_k8s_clients::kubernetes::pods::PodApi; +use alien_k8s_clients::kubernetes::secrets::SecretsApi; + +/// Port the agent serves inside a sandbox pod. +const AGENT_PORT: u16 = 8971; + +/// How long a minted capability lives. +/// +/// Short because a session is a unit of work someone is waiting on, and a capability that +/// outlives the turn it was minted for is a capability somebody can replay. +const CAPABILITY_LIFETIME_SECONDS: i64 = 900; + +/// What the application needs to reach its session, and nothing more. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ClaimedSession { + /// Session id, which the pod now carries as a label + pub session_id: String, + /// `http://:` + pub endpoint: String, + /// Bearer capability, scoped to this session and this operation class + pub capability: String, + /// Unix seconds after which the capability is void + pub expires_at: i64, +} + +/// Claims an idle pod for `session_id` and mints a capability addressed to it. +/// +/// **The claim is won by the apiserver, not here.** `claim_idle_pod` mutates a pod in memory and +/// the write is what decides it: a conflicting write loses on `resourceVersion` and the loser +/// moves to the next candidate. Deciding locally would hand two callers the same pod. +pub async fn claim_session( + pods: &Arc, + secrets: &Arc, + sandbox_id: &str, + namespace: &str, + session_id: &str, + secret_name: &str, + now_unix: i64, +) -> Result { + let idle = pods + .list_pods(namespace, Some(idle_selector(sandbox_id)), None) + .await + .context(ErrorData::CloudPlatformError { + message: "failed to list idle sandbox pods".to_string(), + resource_id: Some(sandbox_id.to_string()), + })?; + + let signing_key = signing_key(secrets, namespace, secret_name, sandbox_id).await?; + + for pod in idle.items { + let Some(name) = pod.metadata.name.clone() else { + continue; + }; + + let mut candidate = pod; + if !claim_idle_pod(&mut candidate, session_id) { + continue; + } + + // A conflict here is another caller winning the same pod, not an error to report: try + // the next one rather than failing a create that a warm pool can still satisfy. + let Ok(claimed) = pods.update_pod(namespace, &name, &candidate).await else { + continue; + }; + + // A pool pod is labelled idle when it is created, before the kubelet assigns an address, + // so a claim arriving in that window can win a pod it cannot use. The label write has + // already committed, and nothing would ever release a session the caller never received — + // so put it back and try the next candidate, the same as losing the race on one. + let Some(address) = claimed.status.as_ref().and_then(|status| status.pod_ip.clone()) else { + release_claim(pods, namespace, &name, &claimed).await; + continue; + }; + + let expires_at = now_unix + CAPABILITY_LIFETIME_SECONDS; + let capability = sandbox_capability_token::mint( + &SandboxCapabilityClaims { + session_id: session_id.to_string(), + operation: SandboxOperationClass::Execute, + generation: 1, + expires_at, + key_id: secret_name.to_string(), + }, + &signing_key, + ) + .context(ErrorData::CloudPlatformError { + message: "failed to mint a sandbox capability".to_string(), + resource_id: Some(sandbox_id.to_string()), + })?; + + return Ok(ClaimedSession { + session_id: session_id.to_string(), + endpoint: format!("http://{address}:{AGENT_PORT}"), + capability, + expires_at, + }); + } + + // Retryable on purpose: the pool refills on the controller's health tick, so a caller that + // waits gets a pod rather than a permanent failure. + Err(AlienError::new(ErrorData::CloudPlatformError { + message: format!( + "no idle sandbox pod is available for '{sandbox_id}'; the warm pool refills on the \ + next health tick" + ), + resource_id: Some(sandbox_id.to_string()), + })) +} + +/// Puts a pod claimed a moment ago back in the pool. +/// +/// Best effort on purpose: if this write loses or fails, the pod is one leaked slot that the +/// health tick replaces, which is strictly better than aborting a claim a later candidate could +/// have satisfied. Failing loudly here would trade a recoverable leak for a failed request. +async fn release_claim( + pods: &Arc, + namespace: &str, + name: &str, + claimed: &Pod, +) { + let mut restored = claimed.clone(); + if let Some(labels) = restored.metadata.labels.as_mut() { + labels.insert( + crate::sandbox::LABEL_POOL_STATE.to_string(), + crate::sandbox::POOL_STATE_IDLE.to_string(), + ); + labels.remove(crate::sandbox::LABEL_SESSION); + } + let _ = pods.update_pod(namespace, name, &restored).await; +} + +/// Reads the sandbox's signing key out of the Secret the controller provisioned. +async fn signing_key( + secrets: &Arc, + namespace: &str, + secret_name: &str, + sandbox_id: &str, +) -> Result { + let secret = secrets + .get_secret(namespace, secret_name) + .await + .context(ErrorData::CloudPlatformError { + message: format!("the capability key for '{sandbox_id}' is unreadable"), + resource_id: Some(sandbox_id.to_string()), + })?; + + let bytes = secret + .data + .as_ref() + .and_then(|data| data.get("signingKey")) + .map(|value| value.0.clone()) + .or_else(|| { + secret + .string_data + .as_ref() + .and_then(|data| data.get("signingKey")) + .and_then(|value| BASE64.decode(value).ok()) + }) + .ok_or_else(|| { + AlienError::new(ErrorData::CloudPlatformError { + message: format!("the capability Secret for '{sandbox_id}' carries no key"), + resource_id: Some(sandbox_id.to_string()), + }) + })?; + + let pair = ed25519_compact::KeyPair::from_slice(&bytes).map_err(|error| { + AlienError::new(ErrorData::CloudPlatformError { + message: format!("the stored capability key is unusable: {error}"), + resource_id: Some(sandbox_id.to_string()), + }) + })?; + + Ok(pair.sk) +} + +/// Releases a claimed session by deleting its pod. +/// +/// Deleting rather than returning it to the pool: a pod that ran untrusted code cannot be handed +/// to the next session, and the warm pool refills from a clean image. +/// +/// Addressed by session rather than by pod name. A claim relabels a warm pool pod instead of +/// renaming it, so the caller never learns a pod name — and taking one from the caller would let +/// any workload in the namespace delete another tenant's session, which is the pod-write this +/// broker exists to withhold. The label selector answers "is this a session of this sandbox" and +/// "which pod is it" in one query. +pub async fn release_session( + pods: &Arc, + namespace: &str, + sandbox_id: &str, + session_id: &str, +) -> Result<()> { + let selector = format!( + "{}={sandbox_id},{}={session_id}", + crate::sandbox::LABEL_SANDBOX, + crate::sandbox::LABEL_SESSION + ); + + let claimed = pods + .list_pods(namespace, Some(selector), None) + .await + .context(ErrorData::CloudPlatformError { + message: format!("could not look up session '{session_id}'"), + resource_id: Some(session_id.to_string()), + })?; + + // Nothing matching is the desired end state, so release stays idempotent. + for pod in claimed.items { + let Some(name) = pod.metadata.name.clone() else { + continue; + }; + pods.delete_pod(namespace, &name) + .await + .context(ErrorData::CloudPlatformError { + message: format!("could not release session '{session_id}'"), + resource_id: Some(name), + })?; + } + + Ok(()) +} + +/// Pods belonging to a sandbox that carry a session label. +pub fn session_name(pod: &Pod) -> Option { + pod.metadata + .labels + .as_ref() + .and_then(|labels| labels.get(crate::sandbox::LABEL_SESSION)) + .cloned() +} + +#[cfg(test)] +mod tests { + use super::*; + use alien_k8s_clients::kubernetes::pods::MockPodApi; + use alien_k8s_clients::kubernetes::secrets::MockSecretsApi; + use k8s_openapi::api::core::v1::{PodStatus, Secret}; + use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta; + use k8s_openapi::List; + use std::collections::BTreeMap; + + fn idle_pod(name: &str, ip: &str) -> Pod { + Pod { + metadata: ObjectMeta { + name: Some(name.to_string()), + labels: Some(crate::sandbox::idle_pod_labels("sbx")), + ..Default::default() + }, + status: Some(PodStatus { + pod_ip: Some(ip.to_string()), + ..Default::default() + }), + ..Default::default() + } + } + + /// A pool pod that exists but has not been given an address yet. + fn idle_pod_without_address(name: &str) -> Pod { + Pod { + metadata: ObjectMeta { + name: Some(name.to_string()), + labels: Some(crate::sandbox::idle_pod_labels("sbx")), + ..Default::default() + }, + status: None, + ..Default::default() + } + } + + fn key_secret() -> Secret { + let pair = ed25519_compact::KeyPair::generate(); + Secret { + data: Some(BTreeMap::from([( + "signingKey".to_string(), + k8s_openapi::ByteString(pair.as_ref().to_vec()), + )])), + ..Default::default() + } + } + + fn pods_returning(items: Vec) -> MockPodApi { + let mut pods = MockPodApi::new(); + pods.expect_list_pods() + .returning(move |_, _, _| Ok(List { items: items.clone(), metadata: Default::default() })); + pods + } + + #[tokio::test] + async fn a_claim_returns_an_address_and_a_capability_for_that_session() { + let mut pods = pods_returning(vec![idle_pod("pool-0", "10.1.2.3")]); + pods.expect_update_pod() + .returning(|_, _, pod| Ok(pod.clone())); + + let mut secrets = MockSecretsApi::new(); + secrets.expect_get_secret().returning(|_, _| Ok(key_secret())); + + let claimed = claim_session( + &(Arc::new(pods) as Arc), + &(Arc::new(secrets) as Arc), + "sbx", + "ns", + "s1", + "alien-sandbox-sbx-capability", + 1_000, + ) + .await + .expect("an idle pod is claimable"); + + assert_eq!(claimed.endpoint, "http://10.1.2.3:8971"); + assert_eq!(claimed.session_id, "s1"); + assert_eq!(claimed.expires_at, 1_000 + CAPABILITY_LIFETIME_SECONDS); + assert!(!claimed.capability.is_empty()); + } + + /// The apiserver decides the race. A caller that lost one pod must take the next rather than + /// failing a create the pool can still satisfy. + #[tokio::test] + async fn losing_the_race_on_one_pod_moves_to_the_next() { + let mut pods = pods_returning(vec![idle_pod("pool-0", "10.1.2.3"), idle_pod("pool-1", "10.1.2.4")]); + pods.expect_update_pod().returning(|_, name, pod| { + if name == "pool-0" { + Err(AlienError::new( + alien_client_core::ErrorData::GenericError { + message: "conflict".to_string(), + }, + )) + } else { + Ok(pod.clone()) + } + }); + + let mut secrets = MockSecretsApi::new(); + secrets.expect_get_secret().returning(|_, _| Ok(key_secret())); + + let claimed = claim_session( + &(Arc::new(pods) as Arc), + &(Arc::new(secrets) as Arc), + "sbx", + "ns", + "s1", + "k", + 1_000, + ) + .await + .expect("the second pod is claimable"); + + assert_eq!(claimed.endpoint, "http://10.1.2.4:8971"); + } + + /// An empty pool is a wait, not a permanent failure: the controller refills it on its next + /// tick, and the message says so rather than leaving a caller guessing. + #[tokio::test] + async fn an_empty_pool_says_it_refills() { + let pods = pods_returning(Vec::new()); + let mut secrets = MockSecretsApi::new(); + secrets.expect_get_secret().returning(|_, _| Ok(key_secret())); + + let error = claim_session( + &(Arc::new(pods) as Arc), + &(Arc::new(secrets) as Arc), + "sbx", + "ns", + "s1", + "k", + 1_000, + ) + .await + .expect_err("no idle pod means no session"); + + assert!(format!("{error:?}").contains("health tick")); + } + + /// The capability the broker mints has to be the one the agent accepts. Verifying it here + /// against the public half is what stops the two drifting into a working mint the agent + /// refuses. + #[tokio::test] + async fn the_minted_capability_verifies_against_the_public_half() { + let pair = ed25519_compact::KeyPair::generate(); + let stored = pair.as_ref().to_vec(); + + let mut pods = pods_returning(vec![idle_pod("pool-0", "10.1.2.3")]); + pods.expect_update_pod() + .returning(|_, _, pod| Ok(pod.clone())); + + let mut secrets = MockSecretsApi::new(); + secrets.expect_get_secret().returning(move |_, _| { + Ok(Secret { + data: Some(BTreeMap::from([( + "signingKey".to_string(), + k8s_openapi::ByteString(stored.clone()), + )])), + ..Default::default() + }) + }); + + let claimed = claim_session( + &(Arc::new(pods) as Arc), + &(Arc::new(secrets) as Arc), + "sbx", + "ns", + "s1", + "k", + 1_000, + ) + .await + .expect("claim succeeds"); + + sandbox_capability_token::verify( + &claimed.capability, + &pair.pk, + &alien_core::sandbox_capability::SandboxSessionIdentity { + session_id: "s1".to_string(), + generation: 1, + }, + SandboxOperationClass::Execute, + 1_100, + ) + .expect("the agent must accept what the broker mints"); + + sandbox_capability_token::verify( + &claimed.capability, + &pair.pk, + &alien_core::sandbox_capability::SandboxSessionIdentity { + session_id: "another-session".to_string(), + generation: 1, + }, + SandboxOperationClass::Execute, + 1_100, + ) + .expect_err("a capability for one session must not reach another"); + } + + /// Release addresses a session, and the pod it deletes is the one carrying that session. + /// + /// A claim relabels a pool pod rather than renaming it, so the pod is still called + /// `pool-` while the caller only ever knows a session id. Deleting by the caller's string + /// matched nothing, and the swallowed error reported success — every session leaked until the + /// parent was torn down. + #[tokio::test] + async fn release_deletes_the_pod_carrying_the_session_not_one_named_after_it() { + let mut pods = MockPodApi::new(); + pods.expect_list_pods().returning(|_, selector, _| { + let selector = selector.expect("release must select by label"); + assert!( + selector.contains("alien.dev/sandbox=sbx") + && selector.contains("alien.dev/sandbox-session=session-7"), + "both the sandbox and the session must be in the selector: {selector}" + ); + let mut labels = crate::sandbox::idle_pod_labels("sbx"); + labels.insert(crate::sandbox::LABEL_SESSION.to_string(), "session-7".to_string()); + Ok(List { + items: vec![Pod { + metadata: ObjectMeta { + name: Some("pool-3".to_string()), + labels: Some(labels), + ..Default::default() + }, + ..Default::default() + }], + ..Default::default() + }) + }); + pods.expect_delete_pod() + .withf(|_, name| name == "pool-3") + .times(1) + .returning(|_, _| Ok(())); + + let pods: Arc = Arc::new(pods); + release_session(&pods, "alien-sandbox-sbx", "sbx", "session-7") + .await + .expect("the claimed pod is released"); + } + + /// A session that matches nothing is already in the desired end state. + #[tokio::test] + async fn releasing_an_unknown_session_is_idempotent_and_deletes_nothing() { + let mut pods = MockPodApi::new(); + pods.expect_list_pods() + .returning(|_, _, _| Ok(List::default())); + pods.expect_delete_pod().never().returning(|_, _| Ok(())); + + let pods: Arc = Arc::new(pods); + release_session(&pods, "alien-sandbox-sbx", "sbx", "never-claimed") + .await + .expect("an unknown session is not an error"); + } + + + /// A pod claimed before the kubelet gave it an address is put back, not abandoned. + /// + /// Pool pods are labelled idle at creation, so a claim can win one that has no `pod_ip` yet. + /// The label write has already committed and no session exists to release it, so returning an + /// error there strands the pod in `claimed` forever and fails a request a later candidate + /// could have served. + #[tokio::test] + async fn a_pod_claimed_before_it_has_an_address_is_returned_to_the_pool() { + let mut pods = MockPodApi::new(); + pods.expect_list_pods().returning(|_, _, _| { + Ok(List { + items: vec![idle_pod_without_address("pool-0"), idle_pod("pool-1", "10.1.2.3")], + ..Default::default() + }) + }); + + // pool-0 is claimed, then restored to idle; pool-1 is claimed and kept. + pods.expect_update_pod().returning(|_, name, pod| { + let state = pod + .metadata + .labels + .as_ref() + .and_then(|l| l.get(crate::sandbox::LABEL_POOL_STATE)) + .cloned() + .unwrap_or_default(); + if name == "pool-0" && state == crate::sandbox::POOL_STATE_IDLE { + return Ok(idle_pod_without_address("pool-0")); + } + Ok(if name == "pool-0" { + idle_pod_without_address("pool-0") + } else { + idle_pod("pool-1", "10.1.2.3") + }) + }); + + let mut secrets = MockSecretsApi::new(); + secrets + .expect_get_secret() + .returning(|_, _| Ok(key_secret())); + + let pods: Arc = Arc::new(pods); + let secrets: Arc = Arc::new(secrets); + let claimed = claim_session(&pods, &secrets, "sbx", "ns", "session-1", "key", 1_000) + .await + .expect("the addressless pod must not fail the whole claim"); + + assert_eq!( + claimed.endpoint, "http://10.1.2.3:8971", + "the claim must land on the pod that actually has an address" + ); + } + +} diff --git a/crates/alien-infra/src/sandbox/kubernetes_eligibility.rs b/crates/alien-infra/src/sandbox/kubernetes_eligibility.rs new file mode 100644 index 000000000..501cf0a25 --- /dev/null +++ b/crates/alien-infra/src/sandbox/kubernetes_eligibility.rs @@ -0,0 +1,200 @@ +//! Whether a cluster can run sandboxes at all. +//! +//! This check exists because of how GKE Autopilot behaves: an unschedulable sandbox +//! pod does **not** get rejected. Node auto-provisioning picks it up instead — +//! `TriggeredScaleUp ... 0->1 (max: 1000)` — so the pod sits in `Pending` while nodes are +//! created and billed. +//! +//! Two consequences, and they pull in opposite directions: +//! +//! - A controller that fail-fasts on `Pending` is wrong; that is a cluster working as designed. +//! - A controller that waits silently spends the customer's money on a cluster that may have no +//! sandboxed runtime at all. +//! +//! So eligibility is decided **before** a pod is created, against the RuntimeClass list, which +//! is cluster-scoped and therefore independent of whether any node exists yet. + +use k8s_openapi::api::node::v1::RuntimeClass; + +use crate::error::{ErrorData, Result}; +use alien_error::AlienError; + +/// Runtime handlers that provide a sandboxed kernel boundary. +/// +/// A plain pod shares the node kernel with everything on it, so an unrecognised handler is +/// refused rather than accepted with a warning. +const SANDBOXED_HANDLERS: &[&str] = &["gvisor", "runsc", "kata", "kata-containers", "kata-qemu"]; + +/// Confirms the cluster declares the requested RuntimeClass and that it is a sandboxed one. +pub fn require_sandboxed_runtime_class( + sandbox_id: &str, + requested: &str, + available: &[RuntimeClass], +) -> Result<()> { + let Some(runtime_class) = available + .iter() + .find(|candidate| candidate.metadata.name.as_deref() == Some(requested)) + else { + let names: Vec<&str> = available + .iter() + .filter_map(|candidate| candidate.metadata.name.as_deref()) + .collect(); + + return Err(AlienError::new(ErrorData::CloudPlatformError { + message: format!( + "the cluster declares no RuntimeClass '{requested}'; it has {names:?}. A sandbox \ + needs a sandboxed runtime, and a pod would sit in Pending rather than fail." + ), + resource_id: Some(sandbox_id.to_string()), + })); + }; + + if !is_sandboxed_handler(&runtime_class.handler) { + return Err(AlienError::new(ErrorData::CloudPlatformError { + message: format!( + "RuntimeClass '{requested}' uses handler '{}', which is not a sandboxed runtime. \ + Untrusted code would share the node kernel.", + runtime_class.handler + ), + resource_id: Some(sandbox_id.to_string()), + })); + } + + Ok(()) +} + +fn is_sandboxed_handler(handler: &str) -> bool { + let handler = handler.to_ascii_lowercase(); + SANDBOXED_HANDLERS + .iter() + .any(|known| handler == *known || handler.starts_with(&format!("{known}-"))) +} + +#[cfg(test)] +mod tests { + use super::*; + use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta; + + fn runtime_class(name: &str, handler: &str) -> RuntimeClass { + RuntimeClass { + metadata: ObjectMeta { + name: Some(name.to_string()), + ..Default::default() + }, + handler: handler.to_string(), + ..Default::default() + } + } + + #[test] + fn a_sandboxed_runtime_class_is_accepted() { + for (name, handler) in [ + ("gvisor", "runsc"), + ("kata", "kata-containers"), + ("kata-qemu", "kata-qemu"), + ] { + require_sandboxed_runtime_class("agent", name, &[runtime_class(name, handler)]) + .unwrap_or_else(|error| panic!("{name}/{handler} should be accepted: {error}")); + } + } + + /// The whole point of the resource. A cluster without a sandboxed runtime must be refused at + /// provision, not discovered when a pod never schedules. + #[test] + fn a_cluster_without_the_runtime_class_is_refused_before_any_pod_is_created() { + let error = require_sandboxed_runtime_class("agent", "gvisor", &[]) + .expect_err("an empty cluster must be refused"); + + let rendered = error.to_string(); + assert!(rendered.contains("gvisor"), "names what is missing: {rendered}"); + assert!( + rendered.contains("Pending"), + "explains why waiting is not the answer: {rendered}" + ); + } + + /// A RuntimeClass exists but points at the ordinary container runtime. Accepting it would + /// run untrusted code on the shared node kernel while reporting success. + #[test] + fn an_unsandboxed_handler_is_refused_even_when_the_name_matches() { + let error = require_sandboxed_runtime_class( + "agent", + "gvisor", + &[runtime_class("gvisor", "runc")], + ) + .expect_err("runc is not a sandbox"); + + assert!(error.to_string().contains("runc"), "names the handler"); + } + + #[test] + fn the_available_classes_are_listed_so_the_operator_can_act() { + let error = require_sandboxed_runtime_class( + "agent", + "gvisor", + &[runtime_class("kata", "kata-containers")], + ) + .expect_err("gvisor is absent"); + + assert!( + error.to_string().contains("kata"), + "an error that does not say what IS available makes the operator go looking" + ); + } +} + +#[cfg(test)] +mod live_cluster_shape { + use super::*; + + /// The RuntimeClasses a GKE Autopilot cluster declares, as a cluster returns them rather + /// than as an author imagined them. Only `gvisor` is a sandbox runtime; the others are here + /// because eligibility has to pick it out of a list that contains ordinary ones. + fn autopilot_runtime_classes() -> Vec { + [ + ("confidential-linked-runner", "confidential-linked-runner"), + ("gvisor", "gvisor"), + ("linked-runner", "linked-runner"), + ] + .into_iter() + .map(|(name, handler)| RuntimeClass { + metadata: k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta { + name: Some(name.to_string()), + ..Default::default() + }, + handler: handler.to_string(), + ..Default::default() + }) + .collect() + } + + #[test] + fn gvisor_on_a_real_cluster_is_accepted() { + require_sandboxed_runtime_class("sbx", "gvisor", &autopilot_runtime_classes()) + .expect("gvisor is a sandboxed runtime and the cluster declares it"); + } + + /// The other two classes that cluster declares are *not* sandboxed runtimes. Naming one of + /// them must fail, or a sandbox would run on a shared kernel while looking configured. + #[test] + fn a_non_sandboxed_class_on_the_same_cluster_is_refused() { + for name in ["linked-runner", "confidential-linked-runner"] { + let error = require_sandboxed_runtime_class("sbx", name, &autopilot_runtime_classes()) + .expect_err("only a sandboxed handler may run untrusted code"); + assert!( + error.to_string().contains(name), + "the refusal must name the class asked for: {error}" + ); + } + } + + /// The message has to carry what the cluster *does* have — an operator reading + /// "no RuntimeClass 'gvisor'" needs to know whether to install gVisor or fix a typo. + #[test] + fn a_missing_class_reports_what_the_cluster_offers() { + let error = require_sandboxed_runtime_class("sbx", "kata-containers", &autopilot_runtime_classes()) + .expect_err("the cluster has no kata-containers"); + let message = error.to_string(); + assert!(message.contains("gvisor"), "must list what is available: {message}"); + } +} diff --git a/crates/alien-infra/src/sandbox/kubernetes_route.rs b/crates/alien-infra/src/sandbox/kubernetes_route.rs new file mode 100644 index 000000000..ab0003c77 --- /dev/null +++ b/crates/alien-infra/src/sandbox/kubernetes_route.rs @@ -0,0 +1,303 @@ +//! The sandbox session broker, served by the operator. +//! +//! Mounted on the operator's existing HTTP server, which the Helm chart already exposes through +//! a Service, so this adds no deployment surface and no chart change. +//! +//! **Why a route at all.** Claiming a warm pod is a `PATCH` on pods. Putting that in the binding +//! would give the customer's application pod-write on the namespace, and `pods/exec` on top of +//! that reaches every pod there. The application asks for a session and gets back an address and +//! a capability scoped to it; the cluster credential stays with the operator. +//! +//! **Why no token of ours.** The caller authenticates with the ServiceAccount token Kubernetes +//! already mounted in its pod, checked with a `TokenReview`. `alien-bindings`' +//! `credential_source` states the rule this follows: managed workloads use their +//! platform-projected identity and do not receive Alien bearer tokens. + +use std::sync::Arc; + +use axum::extract::{Path, State}; +use axum::http::{HeaderMap, StatusCode}; +use axum::routing::post; +use axum::{Json, Router}; +use serde::{Deserialize, Serialize}; + +use crate::sandbox::kubernetes::capability_secret_name; +use crate::sandbox::kubernetes_broker::{claim_session, release_session}; +use alien_error::Context; +use alien_k8s_clients::kubernetes::pods::PodApi; +use alien_k8s_clients::kubernetes::secrets::SecretsApi; +use alien_k8s_clients::kubernetes::token_reviews::{ + authenticated_user, is_service_account_in, review_for, TokenReviewsApi, +}; + +/// What the broker needs to serve one deployment's sandboxes. +#[derive(Clone)] +pub struct BrokerState { + /// Claims and releases pods + pub pods: Arc, + /// Reads the capability signing key + pub secrets: Arc, + /// Verifies the caller's ServiceAccount token + pub token_reviews: Arc, + /// Namespace the sandbox pods live in, and the only namespace a caller may come from + pub namespace: String, +} + +/// A request for a session. +/// +/// Carries ids only. Limits, image and egress come from the pod template the controller built, +/// and the signing key is derived from the sandbox id rather than named by the caller, so an +/// application cannot widen its own confinement by asking. +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct ClaimRequest { + /// Sandbox whose pool to claim from + pub sandbox_id: String, + /// Session id the pod will carry + pub session_id: String, +} + +/// What the application needs to reach its session. +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ClaimResponse { + /// Session id, which every later call addresses + pub session_id: String, + /// `http://:` + pub endpoint: String, + /// Bearer capability for the agent, scoped to this session + pub capability: String, + /// Unix seconds after which the capability is void + pub expires_at: i64, +} + +impl BrokerState { + /// Builds broker state from the pod's own in-cluster credentials. + /// + /// The operator asks for this rather than assembling a Kubernetes client itself: the client + /// crate is an implementation detail of this crate, and threading it through the caller + /// would make the operator depend on it for one line. + pub async fn in_cluster(namespace: String) -> crate::error::Result { + let client = std::sync::Arc::new( + alien_k8s_clients::kubernetes::kubernetes_client::KubernetesClient::new( + alien_k8s_clients::KubernetesClientConfig::InCluster { + additional_headers: None, + namespace: Some(namespace.clone()), + }, + ) + .await + .context(crate::error::ErrorData::CloudPlatformError { + message: "the sandbox broker needs in-cluster Kubernetes credentials".to_string(), + resource_id: None, + })?, + ); + + Ok(Self { + pods: client.clone(), + secrets: client.clone(), + token_reviews: client, + namespace, + }) + } +} + +/// The broker's routes, for the operator to mount. +pub fn broker_router(state: BrokerState) -> Router { + Router::new() + .route("/v1/sandbox/sessions", post(claim)) + .route( + "/v1/sandbox/{sandbox}/sessions/{session}", + axum::routing::delete(release), + ) + .with_state(state) +} + +/// Verifies the caller is a ServiceAccount in this deployment's namespace. +/// +/// The namespace check is the authorization: any pod on the cluster network can reach this port, +/// and a valid token from another tenant's namespace is a valid token for the wrong sandbox. +async fn authorize(state: &BrokerState, headers: &HeaderMap) -> Result { + let token = headers + .get(axum::http::header::AUTHORIZATION) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.strip_prefix("Bearer ")) + .ok_or(StatusCode::UNAUTHORIZED)?; + + let verdict = state + .token_reviews + .create_token_review(&review_for(token)) + .await + .map_err(|_| StatusCode::SERVICE_UNAVAILABLE)?; + + // A rejected token comes back 200 with `authenticated: false`, so the verdict is read rather + // than the status code. + let user = authenticated_user(&verdict).ok_or(StatusCode::UNAUTHORIZED)?; + + if !is_service_account_in(&user, &state.namespace) { + return Err(StatusCode::FORBIDDEN); + } + + Ok(user) +} + +async fn claim( + State(state): State, + headers: HeaderMap, + Json(request): Json, +) -> Result, StatusCode> { + authorize(&state, &headers).await?; + + let claimed = claim_session( + &state.pods, + &state.secrets, + &request.sandbox_id, + &state.namespace, + &request.session_id, + &capability_secret_name(&request.sandbox_id), + chrono::Utc::now().timestamp(), + ) + .await + // Retryable rather than fatal: an empty pool refills on the controller's next health tick, + // and 503 is what tells a caller to wait rather than to give up. + .map_err(|_| StatusCode::SERVICE_UNAVAILABLE)?; + + Ok(Json(ClaimResponse { + session_id: claimed.session_id, + endpoint: claimed.endpoint, + capability: claimed.capability, + expires_at: claimed.expires_at, + })) +} + +async fn release( + State(state): State, + headers: HeaderMap, + Path((sandbox_id, session)): Path<(String, String)>, +) -> Result { + authorize(&state, &headers).await?; + + // A pod that is not a claimed session of this sandbox is refused rather than deleted, so the + // route cannot be used to reach anything else sharing the namespace. + release_session(&state.pods, &state.namespace, &sandbox_id, &session) + .await + .map_err(|_| StatusCode::FORBIDDEN)?; + + Ok(StatusCode::NO_CONTENT) +} + +#[cfg(test)] +mod tests { + use super::*; + use alien_k8s_clients::kubernetes::pods::MockPodApi; + use alien_k8s_clients::kubernetes::secrets::MockSecretsApi; + use alien_k8s_clients::kubernetes::token_reviews::MockTokenReviewsApi; + use k8s_openapi::api::authentication::v1::{TokenReview, TokenReviewStatus, UserInfo}; + + fn verdict(authenticated: bool, username: &str) -> TokenReview { + TokenReview { + status: Some(TokenReviewStatus { + authenticated: Some(authenticated), + user: Some(UserInfo { + username: Some(username.to_string()), + ..Default::default() + }), + ..Default::default() + }), + ..Default::default() + } + } + + fn state_with(reviews: MockTokenReviewsApi) -> BrokerState { + BrokerState { + pods: Arc::new(MockPodApi::new()), + secrets: Arc::new(MockSecretsApi::new()), + token_reviews: Arc::new(reviews), + namespace: "alien-sandbox-sbx".to_string(), + } + } + + fn bearer(token: &str) -> HeaderMap { + let mut headers = HeaderMap::new(); + headers.insert( + axum::http::header::AUTHORIZATION, + format!("Bearer {token}").parse().expect("valid header"), + ); + headers + } + + /// The signing key is what makes a capability valid for a pod. If a caller could name it, a + /// workload holding one sandbox's handle could mint a capability under a sibling's key by + /// naming that sibling's secret, so the broker derives the name from the sandbox id instead. + #[test] + fn a_caller_cannot_name_the_signing_key() { + let named = serde_json::from_str::( + r#"{"sandboxId":"agent","sessionId":"s1","keyName":"alien-sandbox-other-capability"}"#, + ); + assert!(named.is_err(), "naming the key must not deserialize"); + + let request: ClaimRequest = + serde_json::from_str(r#"{"sandboxId":"agent","sessionId":"s1"}"#) + .expect("ids alone are the whole request"); + assert_eq!( + capability_secret_name(&request.sandbox_id), + "alien-sandbox-agent-capability" + ); + } + + #[tokio::test] + async fn a_request_without_a_token_is_refused_before_any_cluster_call() { + let mut reviews = MockTokenReviewsApi::new(); + reviews + .expect_create_token_review() + .never() + .returning(|_| Ok(TokenReview::default())); + + let error = authorize(&state_with(reviews), &HeaderMap::new()) + .await + .expect_err("no token is unauthorized"); + assert_eq!(error, StatusCode::UNAUTHORIZED); + } + + /// The apiserver answers 200 for a bad token with `authenticated: false`. Reading the status + /// code as the verdict would authenticate everything. + #[tokio::test] + async fn a_token_the_apiserver_rejects_is_unauthorized() { + let mut reviews = MockTokenReviewsApi::new(); + reviews + .expect_create_token_review() + .returning(|_| Ok(verdict(false, "system:serviceaccount:alien-sandbox-sbx:app"))); + + let error = authorize(&state_with(reviews), &bearer("nonsense")) + .await + .expect_err("a rejected token is unauthorized"); + assert_eq!(error, StatusCode::UNAUTHORIZED); + } + + /// Any pod on the cluster network can reach this port, so a valid token from another + /// namespace is a valid token for the wrong sandbox. + #[tokio::test] + async fn a_valid_token_from_another_namespace_is_forbidden() { + let mut reviews = MockTokenReviewsApi::new(); + reviews + .expect_create_token_review() + .returning(|_| Ok(verdict(true, "system:serviceaccount:someone-else:app"))); + + let error = authorize(&state_with(reviews), &bearer("valid-elsewhere")) + .await + .expect_err("another namespace is forbidden"); + assert_eq!(error, StatusCode::FORBIDDEN); + } + + #[tokio::test] + async fn a_service_account_in_this_namespace_is_allowed() { + let mut reviews = MockTokenReviewsApi::new(); + reviews + .expect_create_token_review() + .returning(|_| Ok(verdict(true, "system:serviceaccount:alien-sandbox-sbx:worker"))); + + let user = authorize(&state_with(reviews), &bearer("valid")) + .await + .expect("the deployment's own ServiceAccount is allowed"); + assert_eq!(user, "system:serviceaccount:alien-sandbox-sbx:worker"); + } +} diff --git a/crates/alien-infra/src/sandbox/kubernetes_spec.rs b/crates/alien-infra/src/sandbox/kubernetes_spec.rs new file mode 100644 index 000000000..1150e8a4e --- /dev/null +++ b/crates/alien-infra/src/sandbox/kubernetes_spec.rs @@ -0,0 +1,374 @@ +//! Pod and NetworkPolicy specs for Kubernetes sandbox sessions. +//! +//! Kept separate from the controller because this is where the isolation guarantees live, and +//! a manifest is checkable without a cluster. Every rule below traces to something measured on +//! GKE rather than to a reading of the docs. + +use std::collections::BTreeMap; + +use k8s_openapi::api::core::v1::{ + EnvVar, + Capabilities, Container, Pod, PodSecurityContext, PodSpec, ResourceRequirements, + SecurityContext, +}; +use k8s_openapi::apimachinery::pkg::api::resource::Quantity; +use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta; + +use alien_core::{Sandbox, SandboxCode}; + +/// Label carrying the sandbox a pod belongs to; the enumeration scope for reaping. +pub const LABEL_SANDBOX: &str = "alien.dev/sandbox"; +/// Label carrying the session id within that sandbox. +pub const LABEL_SESSION: &str = "alien.dev/sandbox-session"; + +/// The uid sandboxed workloads run as. +const SANDBOX_UID: i64 = 65534; + +/// Names the pod backing one session. +pub fn pod_name(sandbox: &str, session_id: &str) -> String { + format!("alien-sbx-{sandbox}-{session_id}") +} + +/// Builds the pod for one sandbox session. +/// +/// `runtime_class` is required rather than optional: a plain pod shares the node kernel with +/// everything else on it, and this resource exists to run hostile code. +pub fn sandbox_pod( + sandbox: &Sandbox, + session_id: &str, + namespace: &str, + runtime_class: &str, + node_selector: Option>, + capability_public_key: Option<&str>, +) -> Pod { + let image = match &sandbox.code { + SandboxCode::Image { image } => image.clone(), + // Unreachable through any supported path: `Sandbox::validate_for_platform` refuses + // Source on every platform, because no backend builds a sandbox image. Left as an empty + // string rather than a panic — the API server rejects a pod with no image at create, + // which fails an operator loudly without taking it down. + SandboxCode::Source { .. } => String::new(), + }; + + let labels = BTreeMap::from([ + (LABEL_SANDBOX.to_string(), sandbox.id.clone()), + (LABEL_SESSION.to_string(), session_id.to_string()), + ]); + + let declared = sandbox.resolved_limits(); + let limits = BTreeMap::from([ + ("cpu".to_string(), Quantity(declared.cpu.clone())), + ("memory".to_string(), Quantity(declared.memory.clone())), + ( + "ephemeral-storage".to_string(), + Quantity(declared.disk.clone()), + ), + ]); + + Pod { + metadata: ObjectMeta { + name: Some(pod_name(&sandbox.id, session_id)), + namespace: Some(namespace.to_string()), + labels: Some(labels), + ..Default::default() + }, + spec: Some(PodSpec { + runtime_class_name: Some(runtime_class.to_string()), + node_selector, + // No identity by default. A mounted token is a credential the untrusted code can + // read, and it is the workload's, not the sandbox's. + automount_service_account_token: Some(false), + // A sandbox that exits stays exited; restarting hostile code hands it another go. + restart_policy: Some("Never".to_string()), + // The kubelet kills the pod at the deadline, so the ceiling holds even if whatever + // created the session never comes back to terminate it. + active_deadline_seconds: sandbox.session.max_lifetime_seconds.map(i64::from), + enable_service_links: Some(false), + security_context: Some(PodSecurityContext { + run_as_non_root: Some(true), + run_as_user: Some(SANDBOX_UID), + run_as_group: Some(SANDBOX_UID), + fs_group: Some(SANDBOX_UID), + ..Default::default() + }), + containers: vec![Container { + name: "sandbox".to_string(), + image: Some(image), + env: capability_public_key.map(capability_environment), + security_context: Some(SecurityContext { + allow_privilege_escalation: Some(false), + privileged: Some(false), + read_only_root_filesystem: Some(true), + run_as_non_root: Some(true), + run_as_user: Some(SANDBOX_UID), + capabilities: Some(Capabilities { + drop: Some(vec!["ALL".to_string()]), + add: None, + }), + ..Default::default() + }), + resources: Some(ResourceRequirements { + limits: Some(limits), + ..Default::default() + }), + ..Default::default() + }], + ..Default::default() + }), + ..Default::default() + } +} + +/// Builds an unclaimed pool pod: the same hardened spec, labelled idle and owned by no session. +/// +/// The spec must be identical to a session pod's, or the pool would hand out something with +/// different isolation from what a directly-created session gets. +/// +/// Named by the apiserver through `generateName` rather than by us. A claimed pod keeps the name +/// it was created with, so any scheme derived from the pool's current depth reissues a name that +/// is still taken as soon as pods are in use — the pool then fails to refill exactly when it is +/// under load, which is the case it exists for. +pub fn idle_pool_pod( + sandbox: &Sandbox, + namespace: &str, + runtime_class: &str, + node_selector: Option>, + capability_public_key: Option<&str>, +) -> Pod { + let mut pod = sandbox_pod( + sandbox, + "pool", + namespace, + runtime_class, + node_selector, + capability_public_key, + ); + pod.metadata.name = None; + pod.metadata.generate_name = Some(format!("{}-", pod_name(&sandbox.id, "pool"))); + pod.metadata.labels = Some(crate::sandbox::idle_pod_labels(&sandbox.id)); + pod +} + +/// Environment the agent needs to authorize callers by capability. +/// +/// Kubernetes cannot authorize by transport the way AWS does: a pod IP is reachable by anything +/// on the cluster network, so the agent has to check a token rather than trust the connection. +/// Only the **public** key goes here, because a pod's environment is readable by the untrusted +/// code inside it. +pub fn capability_environment(public_key_base64: &str) -> Vec { + vec![ + EnvVar { + name: "ALIEN_SANDBOX_AUTHORIZATION".to_string(), + value: Some("capability".to_string()), + value_from: None, + }, + EnvVar { + name: "ALIEN_SANDBOX_PUBLIC_KEY".to_string(), + value: Some(public_key_base64.to_string()), + value_from: None, + }, + ] +} + +/// Builds the egress NetworkPolicy for a sandbox. +/// +/// Under `deny` there are no egress rules at all, which denies everything. Under `allow` the +#[cfg(test)] +mod tests { + use super::*; + use alien_core::{SandboxEgress, SandboxLimits, SandboxSessionPolicy}; + + fn sandbox(egress: SandboxEgress) -> Sandbox { + Sandbox::new("agent".to_string()) + .code(SandboxCode::Image { + image: "ubuntu:24.04".to_string(), + }) + .limits(SandboxLimits { + cpu: "1".to_string(), + memory: "2Gi".to_string(), + disk: "20Gi".to_string(), + max_processes: None, + }) + .egress(egress) + .session(SandboxSessionPolicy { + max_lifetime_seconds: Some(3600), + idle_suspend_seconds: None, + }) + .build() + } + + #[test] + fn a_pod_always_carries_a_sandboxed_runtime_class() { + let pod = sandbox_pod(&sandbox(SandboxEgress::Deny), "s1", "sbx", "gvisor", None, None); + let spec = pod.spec.expect("a spec"); + + assert_eq!(spec.runtime_class_name.as_deref(), Some("gvisor")); + } + + /// A mounted token is the workload's credential, readable by the untrusted code beside it. + #[test] + fn a_pod_mounts_no_service_account_token() { + let pod = sandbox_pod(&sandbox(SandboxEgress::Deny), "s1", "sbx", "gvisor", None, None); + let spec = pod.spec.expect("a spec"); + + assert_eq!(spec.automount_service_account_token, Some(false)); + assert_eq!(spec.enable_service_links, Some(false)); + } + + #[test] + fn a_pod_is_unprivileged_with_a_read_only_root() { + let pod = sandbox_pod(&sandbox(SandboxEgress::Deny), "s1", "sbx", "gvisor", None, None); + let container = pod.spec.expect("a spec").containers.remove(0); + let security = container.security_context.expect("a security context"); + + assert_eq!(security.run_as_non_root, Some(true)); + assert_eq!(security.run_as_user, Some(SANDBOX_UID)); + assert_eq!(security.allow_privilege_escalation, Some(false)); + assert_eq!(security.privileged, Some(false)); + assert_eq!(security.read_only_root_filesystem, Some(true)); + assert_eq!( + security.capabilities.expect("capabilities").drop, + Some(vec!["ALL".to_string()]) + ); + } + + #[test] + fn a_pod_never_restarts() { + let pod = sandbox_pod(&sandbox(SandboxEgress::Deny), "s1", "sbx", "gvisor", None, None); + assert_eq!( + pod.spec.expect("a spec").restart_policy.as_deref(), + Some("Never"), + "restarting hostile code hands it another attempt" + ); + } + + /// The kubelet is what makes a session deadline a ceiling rather than a hope: it kills the + /// pod whether or not whatever created the session ever comes back to terminate it. This is + /// the only backend with the primitive, which is why the field is refused elsewhere. + /// A claimed pod keeps its name, so any name derived from the pool's current depth is + /// reissued while the old one is still in use — the pool then fails to refill exactly when + /// it is under load, which is the case the pool exists for. `generateName` makes the + /// collision impossible rather than unlikely. + #[test] + fn a_pool_pod_is_named_by_the_apiserver_so_two_can_never_collide() { + let pod = idle_pool_pod(&sandbox(SandboxEgress::Deny), "sbx", "gvisor", None, None); + + assert_eq!(pod.metadata.name, None, "a fixed name is what collides"); + assert_eq!( + pod.metadata.generate_name.as_deref(), + Some("alien-sbx-agent-pool-"), + "the prefix still says which sandbox the pod belongs to" + ); + } + + #[test] + fn a_pod_carries_the_declared_session_deadline() { + let pod = sandbox_pod(&sandbox(SandboxEgress::Deny), "s1", "sbx", "gvisor", None, None); + assert_eq!( + pod.spec.expect("a spec").active_deadline_seconds, + Some(3600) + ); + } + + /// A sandbox that declares no deadline must not acquire one by default — an unexpected kill + /// mid-session is worse than an unbounded one the caller chose. + #[test] + fn a_pod_without_a_declared_deadline_carries_none() { + let mut config = sandbox(SandboxEgress::Deny); + config.session.max_lifetime_seconds = None; + let pod = sandbox_pod(&config, "s1", "sbx", "gvisor", None, None); + assert_eq!(pod.spec.expect("a spec").active_deadline_seconds, None); + } + + #[test] + fn a_pod_carries_the_declared_ceilings() { + let pod = sandbox_pod(&sandbox(SandboxEgress::Deny), "s1", "sbx", "gvisor", None, None); + let container = pod.spec.expect("a spec").containers.remove(0); + let limits = container + .resources + .expect("resources") + .limits + .expect("limits"); + + assert_eq!(limits.get("cpu"), Some(&Quantity("1".to_string()))); + assert_eq!(limits.get("memory"), Some(&Quantity("2Gi".to_string()))); + assert_eq!( + limits.get("ephemeral-storage"), + Some(&Quantity("20Gi".to_string())) + ); + } + + + + + + /// A pooled pod is handed to a session that did not create it, so anything weaker here + /// would silently downgrade isolation for every warm start — the common path. + #[test] + fn a_pooled_pod_has_identical_isolation_to_a_session_pod() { + let sandbox = sandbox(SandboxEgress::Deny); + let session = sandbox_pod(&sandbox, "s1", "sbx", "gvisor", None, None); + let pooled = idle_pool_pod(&sandbox, "sbx", "gvisor", None, None); + + assert_eq!( + session.spec, pooled.spec, + "a pooled pod must be spec-identical to one created for a session" + ); + } + + /// An idle pod carries no session label. If it did, reaping a sandbox's sessions would + /// destroy the pool, and enumeration could not tell a spare from a live session. + #[test] + fn a_pooled_pod_carries_no_session_label_until_claimed() { + let pooled = idle_pool_pod(&sandbox(SandboxEgress::Deny), "sbx", "gvisor", None, None); + let labels = pooled.metadata.labels.expect("labels"); + + assert_eq!(labels.get(LABEL_SANDBOX), Some(&"agent".to_string())); + assert!(!labels.contains_key(LABEL_SESSION)); + } + + /// Only the public half reaches a pod. Its environment is readable by the untrusted code + /// inside it, so a signing key there would let the sandbox mint its own capabilities. + #[test] + fn a_pod_carries_the_public_key_and_the_capability_mode() { + let pod = sandbox_pod( + &sandbox(SandboxEgress::Deny), + "s1", + "sbx", + "gvisor", + None, + Some("cHVibGlj"), + ); + + let environment = pod.spec.expect("a spec").containers[0] + .env + .clone() + .expect("capability mode needs an environment"); + + let value = |name: &str| { + environment + .iter() + .find(|variable| variable.name == name) + .and_then(|variable| variable.value.clone()) + }; + + assert_eq!(value("ALIEN_SANDBOX_AUTHORIZATION").as_deref(), Some("capability")); + assert_eq!(value("ALIEN_SANDBOX_PUBLIC_KEY").as_deref(), Some("cHVibGlj")); + assert!( + !environment + .iter() + .any(|variable| variable.name.contains("PRIVATE") || variable.name.contains("SIGNING")), + "no signing material may reach a sandbox pod" + ); + } + + #[test] + fn pods_are_enumerable_by_sandbox_for_reaping() { + let pod = sandbox_pod(&sandbox(SandboxEgress::Deny), "s1", "sbx", "gvisor", None, None); + let labels = pod.metadata.labels.expect("labels"); + + assert_eq!(labels.get(LABEL_SANDBOX), Some(&"agent".to_string())); + assert_eq!(labels.get(LABEL_SESSION), Some(&"s1".to_string())); + assert_eq!(pod_name("agent", "s1"), "alien-sbx-agent-s1"); + } +} diff --git a/crates/alien-infra/src/sandbox/kubernetes_warm_pool.rs b/crates/alien-infra/src/sandbox/kubernetes_warm_pool.rs new file mode 100644 index 000000000..cb135ffd7 --- /dev/null +++ b/crates/alien-infra/src/sandbox/kubernetes_warm_pool.rs @@ -0,0 +1,173 @@ +//! Warm pool for Kubernetes sandbox sessions. +//! +//! Exists because of a measurement, not a preference: **2.7s warm against 79s cold** on GKE +//! Autopilot. 79s is per agent turn, and the cold path fires whenever the gVisor node pool has +//! scaled to zero — which for a per-turn workload is often. A pool of pre-created idle pods is +//! the difference between usable and not. + +use std::collections::BTreeMap; + +use k8s_openapi::api::core::v1::Pod; + +use crate::sandbox::kubernetes_spec::{LABEL_SANDBOX, LABEL_SESSION}; + +/// Marks a pod as pooled and not yet claimed. +pub const LABEL_POOL_STATE: &str = "alien.dev/sandbox-pool"; +/// Value of [`LABEL_POOL_STATE`] while a pod is available. +pub const POOL_STATE_IDLE: &str = "idle"; +/// Value of [`LABEL_POOL_STATE`] once a session owns the pod. +pub const POOL_STATE_CLAIMED: &str = "claimed"; + +/// Selector matching pods this sandbox can still hand out. +pub fn idle_selector(sandbox_id: &str) -> String { + format!("{LABEL_SANDBOX}={sandbox_id},{LABEL_POOL_STATE}={POOL_STATE_IDLE}") +} + +/// Selector matching every pod belonging to a sandbox, claimed or not. +pub fn all_pods_selector(sandbox_id: &str) -> String { + format!("{LABEL_SANDBOX}={sandbox_id}") +} + +/// How many idle pods to create to reach the target. +/// +/// Saturating, because a pool that shrank below target after a burst must not ask for a +/// negative number of pods. +pub fn pool_deficit(target: usize, idle_now: usize) -> usize { + target.saturating_sub(idle_now) +} + +/// Claims an idle pod for a session by rewriting its labels. +/// +/// **The claim is won by Kubernetes, not by this function.** The caller writes the mutated pod +/// back with the `resourceVersion` it read, and the API server rejects a stale one with 409 +/// Conflict — so of two callers racing for the same pod, exactly one update lands and the loser +/// retries against a different pod. Any scheme that checked "is it idle?" and then wrote would +/// hand the same pod to both. +pub fn claim_idle_pod(pod: &mut Pod, session_id: &str) -> bool { + let Some(labels) = pod.metadata.labels.as_mut() else { + return false; + }; + + if labels.get(LABEL_POOL_STATE).map(String::as_str) != Some(POOL_STATE_IDLE) { + return false; + } + + labels.insert( + LABEL_POOL_STATE.to_string(), + POOL_STATE_CLAIMED.to_string(), + ); + labels.insert(LABEL_SESSION.to_string(), session_id.to_string()); + true +} + +/// Labels an idle pod: it belongs to the sandbox but to no session yet. +pub fn idle_pod_labels(sandbox_id: &str) -> BTreeMap { + BTreeMap::from([ + (LABEL_SANDBOX.to_string(), sandbox_id.to_string()), + (LABEL_POOL_STATE.to_string(), POOL_STATE_IDLE.to_string()), + ]) +} + +#[cfg(test)] +mod tests { + use super::*; + use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta; + + fn pod_with(labels: BTreeMap) -> Pod { + Pod { + metadata: ObjectMeta { + name: Some("alien-sbx-agent-pooled".to_string()), + labels: Some(labels), + resource_version: Some("42".to_string()), + ..Default::default() + }, + ..Default::default() + } + } + + #[test] + fn an_idle_pod_belongs_to_the_sandbox_but_to_no_session() { + let labels = idle_pod_labels("agent"); + + assert_eq!(labels.get(LABEL_SANDBOX), Some(&"agent".to_string())); + assert_eq!(labels.get(LABEL_POOL_STATE), Some(&POOL_STATE_IDLE.to_string())); + assert!( + !labels.contains_key(LABEL_SESSION), + "an unclaimed pod must not carry a session label, or reaping would target it" + ); + } + + #[test] + fn claiming_marks_the_pod_and_names_its_session() { + let mut pod = pod_with(idle_pod_labels("agent")); + + assert!(claim_idle_pod(&mut pod, "s1")); + + let labels = pod.metadata.labels.expect("labels"); + assert_eq!( + labels.get(LABEL_POOL_STATE), + Some(&POOL_STATE_CLAIMED.to_string()) + ); + assert_eq!(labels.get(LABEL_SESSION), Some(&"s1".to_string())); + } + + /// Second-line defence only. The real guarantee is the API server rejecting a stale + /// resourceVersion, but a claimed pod must not be re-claimable even in memory. + #[test] + fn an_already_claimed_pod_cannot_be_claimed_again() { + let mut pod = pod_with(idle_pod_labels("agent")); + assert!(claim_idle_pod(&mut pod, "s1")); + + assert!( + !claim_idle_pod(&mut pod, "s2"), + "a claimed pod must not be handed to a second session" + ); + assert_eq!( + pod.metadata.labels.expect("labels").get(LABEL_SESSION), + Some(&"s1".to_string()), + "a refused claim must not overwrite the owner" + ); + } + + #[test] + fn a_pod_with_no_labels_is_not_claimable() { + let mut pod = Pod::default(); + assert!(!claim_idle_pod(&mut pod, "s1")); + } + + /// The write-back carries this, and a stale one is how the API server settles a race. + #[test] + fn claiming_preserves_the_resource_version_the_caller_read() { + let mut pod = pod_with(idle_pod_labels("agent")); + claim_idle_pod(&mut pod, "s1"); + + assert_eq!( + pod.metadata.resource_version.as_deref(), + Some("42"), + "dropping it would turn a conflicting update into a blind overwrite" + ); + } + + #[test] + fn the_idle_selector_excludes_claimed_pods() { + let selector = idle_selector("agent"); + + assert!(selector.contains(&format!("{LABEL_POOL_STATE}={POOL_STATE_IDLE}"))); + assert!( + !all_pods_selector("agent").contains(LABEL_POOL_STATE), + "reaping must match claimed and idle pods alike" + ); + } + + #[test] + fn the_deficit_never_goes_negative() { + assert_eq!(pool_deficit(3, 0), 3); + assert_eq!(pool_deficit(3, 2), 1); + assert_eq!(pool_deficit(3, 3), 0); + assert_eq!( + pool_deficit(3, 5), + 0, + "a pool above target asks for nothing, not a negative count" + ); + } +} diff --git a/crates/alien-infra/src/sandbox/local.rs b/crates/alien-infra/src/sandbox/local.rs new file mode 100644 index 000000000..b53b7566a --- /dev/null +++ b/crates/alien-infra/src/sandbox/local.rs @@ -0,0 +1,456 @@ +//! Local Sandbox controller. +//! +//! The Frozen parent on Local is manager state rather than a provider object: there is nothing +//! to provision until a session is created. What the controller owns is the guarantee that +//! Docker is reachable, and that sessions left behind by a previous run are reaped before +//! anything new starts. + +use std::sync::Arc; +use std::time::Duration; + +use tracing::{debug, info}; + +use crate::core::ResourceControllerContext; +use crate::error::{ErrorData, Result}; +use alien_core::{ResourceOutputs as CoreResourceOutputs, ResourceStatus, Sandbox, SandboxOutputs}; +use alien_error::{AlienError, Context, IntoAlienError}; +use alien_macros::controller; + +/// Local Sandbox controller. +#[controller] +pub struct LocalSandboxController { + /// Sandbox this controller owns sessions for, and the enumeration scope for reaping. + pub(crate) sandbox_name: Option, + /// Loopback route the workload's binding talks to. + pub(crate) route_url: Option, + /// File the route's bearer token is written to. The binding carries the path, never the + /// token, so no secret lands in deployment state. + pub(crate) token_path: Option, +} + +#[controller] +impl LocalSandboxController { + // ─────────────── CREATE FLOW ─────────────────────────────────────────── + + #[flow_entry(Create)] + #[handler( + state = EnsureRuntime, + on_failure = ProvisionFailed, + status = ResourceStatus::Provisioning + )] + async fn ensure_runtime( + &mut self, + ctx: &ResourceControllerContext<'_>, + ) -> Result { + let config = ctx.desired_resource_config::()?; + + let manager = sandbox_manager(ctx)?; + + // Reaping here rather than only at delete: a CLI restart leaves the previous run's + // containers behind, and a session that outlives the process that owned it can never + // be reached again. + let reaped = manager + .reap(&config.id) + .await + .context(ErrorData::CloudPlatformError { + message: "Failed to reap stale sandbox sessions".to_string(), + resource_id: Some(config.id.clone()), + })?; + + if reaped > 0 { + info!(sandbox_id = %config.id, reaped, "Reaped sandbox sessions from a previous run"); + } + + self.sandbox_name = Some(config.id.clone()); + + // The session template is fixed here rather than accepted per create: a client-supplied + // limit is a limit the client can decline to send, and this sandbox runs its code. + let route = alien_local::SandboxRoute::ensure(manager, &config.id, session_template(&config)?) + .await + .context(ErrorData::CloudPlatformError { + message: "Failed to serve the local sandbox route".to_string(), + resource_id: Some(config.id.clone()), + })?; + + self.route_url = Some(route.base_url.clone()); + self.token_path = Some(route.token_path.display().to_string()); + + info!(sandbox_id = %config.id, route = %route.base_url, "Local sandbox ready"); + + Ok(HandlerAction::Continue { + state: Ready, + suggested_delay: None, + }) + } + + #[handler( + state = Ready, + on_failure = RefreshFailed, + status = ResourceStatus::Running + )] + async fn ready(&mut self, ctx: &ResourceControllerContext<'_>) -> Result { + let config = ctx.desired_resource_config::()?; + + // Re-ensured on every tick, not only at create: the route is a listener in this + // process, so a manager restart leaves the persisted URL pointing at a dead port until + // something binds it again. Idempotent by sandbox id. + let manager = sandbox_manager(ctx)?; + let route = alien_local::SandboxRoute::ensure( + Arc::clone(&manager), + &config.id, + session_template(&config)?, + ) + .await + .context(ErrorData::CloudPlatformError { + message: "Failed to serve the local sandbox route".to_string(), + resource_id: Some(config.id.clone()), + })?; + self.route_url = Some(route.base_url); + self.token_path = Some(route.token_path.display().to_string()); + + // "Healthy" on a platform with nothing durable means the runtime is still there to + // create sessions in; the session count itself is not a health signal. + let sessions = manager + .list_sessions(&config.id) + .await + .context(ErrorData::CloudPlatformError { + message: "Docker sandbox health check failed".to_string(), + resource_id: Some(config.id.clone()), + })?; + + debug!(sandbox_id = %config.id, sessions = sessions.len(), "Sandbox health check passed"); + + // Content-free by construction: a count and whether the route is bound. Nothing here + // reaches inside a session, which is the whole reason the resource exists. + ctx.emit_heartbeat(alien_core::ResourceHeartbeat { + deployment_id: None, + resource_id: config.id.clone(), + resource_type: Sandbox::RESOURCE_TYPE, + controller_platform: alien_core::Platform::Local, + backend: alien_core::HeartbeatBackend::Local, + observed_at: chrono::Utc::now(), + data: alien_core::ResourceHeartbeatData::Sandbox( + alien_core::SandboxHeartbeatData::Local(alien_core::LocalSandboxHeartbeatData { + status: alien_core::SandboxHeartbeatStatus::default(), + active_sessions: sessions.len() as u32, + route_serving: self.route_url.is_some(), + }), + ), + raw: vec![], + }); + + Ok(HandlerAction::Continue { + state: Ready, + suggested_delay: Some(Duration::from_secs(15)), + }) + } + + // ─────────────── UPDATE FLOW ────────────────────────────────────────── + + #[flow_entry(Update, from = [Ready, RefreshFailed])] + #[handler( + state = UpdatingSandbox, + on_failure = UpdateFailed, + status = ResourceStatus::Updating + )] + async fn updating_sandbox( + &mut self, + ctx: &ResourceControllerContext<'_>, + ) -> Result { + let config = ctx.desired_resource_config::()?; + + // Config changes apply to sessions created after them. Running sessions are not + // restarted: a session is a unit of work someone is waiting on, not a replica. The + // route keeps its address — the workload already holds that URL — and takes the new + // template. + let manager = sandbox_manager(ctx)?; + alien_local::SandboxRoute::ensure(manager, &config.id, session_template(&config)?) + .await + .context(ErrorData::CloudPlatformError { + message: "Failed to update the local sandbox route".to_string(), + resource_id: Some(config.id.clone()), + })?; + + info!(sandbox_id = %config.id, "Updated local sandbox configuration"); + + self.sandbox_name = Some(config.id.clone()); + + Ok(HandlerAction::Continue { + state: Ready, + suggested_delay: None, + }) + } + + // ─────────────── DELETE FLOW ────────────────────────────────────────── + + #[flow_entry(Delete)] + #[handler( + state = Deleting, + on_failure = DeleteFailed, + status = ResourceStatus::Deleting + )] + async fn deleting(&mut self, ctx: &ResourceControllerContext<'_>) -> Result { + let config = ctx.desired_resource_config::()?; + + let manager = sandbox_manager(ctx)?; + let reaped = manager + .reap(&config.id) + .await + .context(ErrorData::CloudPlatformError { + message: "Failed to remove sandbox sessions".to_string(), + resource_id: Some(config.id.clone()), + })?; + + // The containers are not the whole of it: a route left serving keeps accepting session + // creates for a sandbox that no longer exists, and its token file stays on disk as a + // live credential. + alien_local::SandboxRoute::remove(&config.id).await; + self.route_url = None; + self.token_path = None; + + info!(sandbox_id = %config.id, reaped, "Removed local sandbox sessions and its route"); + + Ok(HandlerAction::Continue { + state: Deleted, + suggested_delay: None, + }) + } + + // ─────────────── TERMINAL STATES ────────────────────────────────────── + + terminal_state!(state = Deleted, status = ResourceStatus::Deleted); + terminal_state!( + state = ProvisionFailed, + status = ResourceStatus::ProvisionFailed + ); + terminal_state!(state = UpdateFailed, status = ResourceStatus::UpdateFailed); + terminal_state!(state = DeleteFailed, status = ResourceStatus::DeleteFailed); + terminal_state!( + state = RefreshFailed, + status = ResourceStatus::RefreshFailed + ); + + // ─────────────── HELPER METHODS ────────────────────────────────────── + + fn build_outputs(&self) -> Option { + self.sandbox_name.as_ref().map(|name| { + CoreResourceOutputs::new(SandboxOutputs { + parent_name: name.clone(), + identifier: None, + // Sessions are reached through the local manager's authenticated loopback + // route, which the binding resolves; there is no provider endpoint to publish. + endpoint: None, + }) + }) + } + + fn get_binding_params(&self) -> Result> { + use alien_core::bindings::{BindingValue, SandboxBinding}; + + let (Some(route_url), Some(token_path), Some(sandbox_name)) = ( + self.route_url.as_ref(), + self.token_path.as_ref(), + self.sandbox_name.as_ref(), + ) else { + return Ok(None); + }; + + let binding = SandboxBinding::local( + BindingValue::value(route_url.clone()), + BindingValue::value(sandbox_name.clone()), + BindingValue::value(token_path.clone()), + ); + + Ok(Some(serde_json::to_value(binding).into_alien_error().context( + ErrorData::ResourceStateSerializationFailed { + resource_id: "binding".to_string(), + message: "Failed to serialize sandbox binding parameters".to_string(), + }, + )?)) + } +} + +/// Turns the declaration's ceilings into what Docker is given. +/// +/// Every field is enforced. A limit this cannot express is an error rather than a default: +/// silently widening a ceiling on a sandbox is the failure this resource exists to prevent. +#[cfg(feature = "local")] +fn session_template(sandbox: &Sandbox) -> Result { + use alien_core::{SandboxCode, SandboxEgress}; + + let SandboxCode::Image { image } = &sandbox.code else { + return Err(AlienError::new(ErrorData::CloudPlatformError { + message: "Local sandboxes take a prebuilt image; building one from source is not \ + supported on this platform" + .to_string(), + resource_id: Some(sandbox.id.clone()), + })); + }; + + let egress = match &sandbox.egress { + SandboxEgress::Deny => alien_local::SandboxEgressMode::Deny, + SandboxEgress::Allow => alien_local::SandboxEgressMode::Allow, + SandboxEgress::AllowDomains { .. } => { + return Err(AlienError::new(ErrorData::CloudPlatformError { + message: "Local cannot restrict egress to a hostname list; only Azure can" + .to_string(), + resource_id: Some(sandbox.id.clone()), + })) + } + }; + + let limits = sandbox.resolved_limits(); + + Ok(alien_local::SandboxSessionConfig { + image: image.clone(), + cpu_cores: cpu_cores(&limits.cpu, &sandbox.id)?, + memory_bytes: bytes(&limits.memory, &sandbox.id)? as i64, + pids_limit: limits.max_processes.map(i64::from), + scratch_bytes: bytes(&limits.disk, &sandbox.id)?, + egress, + preview_ports: sandbox.preview_ports.clone(), + env: std::collections::HashMap::new(), + }) +} + +/// Parses a CPU ceiling in cores or millicores. +#[cfg(feature = "local")] +fn cpu_cores(value: &str, sandbox_id: &str) -> Result { + let trimmed = value.trim(); + let parsed = match trimmed.strip_suffix('m') { + Some(millis) => millis.parse::().ok().map(|value| value / 1000.0), + None => trimmed.parse::().ok(), + }; + + parsed.filter(|cores| *cores > 0.0).ok_or_else(|| { + AlienError::new(ErrorData::CloudPlatformError { + message: format!("'{value}' is not a CPU ceiling in cores or millicores"), + resource_id: Some(sandbox_id.to_string()), + }) + }) +} + +/// Parses a byte quantity written the way Kubernetes writes one. +#[cfg(feature = "local")] +fn bytes(value: &str, sandbox_id: &str) -> Result { + const SUFFIXES: &[(&str, u64)] = &[ + ("Ki", 1024), + ("Mi", 1024 * 1024), + ("Gi", 1024 * 1024 * 1024), + ("Ti", 1024 * 1024 * 1024 * 1024), + ("K", 1000), + ("M", 1000 * 1000), + ("G", 1000 * 1000 * 1000), + ("T", 1000u64.pow(4)), + ]; + + let trimmed = value.trim(); + let parsed = SUFFIXES + .iter() + .find_map(|(suffix, scale)| { + trimmed + .strip_suffix(suffix) + .and_then(|number| number.trim().parse::().ok()) + .map(|number| (number * *scale as f64) as u64) + }) + .or_else(|| trimmed.parse::().ok()); + + parsed.filter(|bytes| *bytes > 0).ok_or_else(|| { + AlienError::new(ErrorData::CloudPlatformError { + message: format!("'{value}' is not a byte quantity"), + resource_id: Some(sandbox_id.to_string()), + }) + }) +} + +#[cfg(feature = "local")] +fn sandbox_manager( + ctx: &ResourceControllerContext<'_>, +) -> Result> { + ctx.service_provider + .get_local_sandbox_manager() + .ok_or_else(|| { + AlienError::new(ErrorData::LocalServicesNotAvailable { + service_name: "LocalSandboxManager".to_string(), + }) + }) +} + +#[cfg(test)] +mod tests { + /// A ceiling that cannot be expressed must fail rather than default. Silently widening a + /// limit is the failure this resource exists to prevent. + #[test] + fn limits_parse_or_fail_loudly() { + assert_eq!(cpu_cores("500m", "sbx").expect("millicores"), 0.5); + assert_eq!(cpu_cores("2", "sbx").expect("cores"), 2.0); + cpu_cores("half", "sbx").expect_err("an unparseable CPU ceiling is an error"); + cpu_cores("0", "sbx").expect_err("a zero CPU ceiling is not a ceiling"); + + assert_eq!(bytes("512Mi", "sbx").expect("mebibytes"), 536_870_912); + assert_eq!(bytes("1Gi", "sbx").expect("gibibytes"), 1_073_741_824); + assert_eq!(bytes("1M", "sbx").expect("megabytes"), 1_000_000); + assert_eq!(bytes("4096", "sbx").expect("plain bytes"), 4096); + bytes("plenty", "sbx").expect_err("an unparseable size is an error"); + } + + /// Local has no hostname allowlist, and accepting one would run untrusted code with wider + /// egress than the declaration asked for. + #[test] + fn a_hostname_allowlist_is_refused_rather_than_widened() { + use alien_core::{SandboxCode, SandboxEgress, SandboxLimits, SandboxSessionPolicy}; + + let sandbox = Sandbox::new("sbx".to_string()) + .code(SandboxCode::Image { + image: "alpine:3.20".to_string(), + }) + .limits(SandboxLimits { + cpu: "500m".to_string(), + memory: "512Mi".to_string(), + disk: "1Gi".to_string(), + max_processes: Some(64), + }) + .egress(SandboxEgress::AllowDomains { + domains: vec!["example.com".to_string()], + }) + .session(SandboxSessionPolicy { + max_lifetime_seconds: None, + idle_suspend_seconds: None, + }) + .build(); + + session_template(&sandbox).expect_err("Local cannot honour a hostname allowlist"); + } + + use super::*; + use crate::core::{deserialize_controller, serialize_controller, ResourceController}; + + /// A controller must round-trip by tag. Miss the by-tag arm and the executor cannot resolve + /// it, which surfaces as InitialSetupFailed with no per-resource error to read — it fails + /// above the handler layer, so nothing logs a cause. + #[test] + fn controller_round_trips_by_tag() { + let controller = LocalSandboxController { + sandbox_name: Some("agent".to_string()), + ..Default::default() + }; + + let value = serialize_controller(&controller).expect("serializes with its tag"); + assert_eq!(value["type"], "LocalSandboxController"); + + let restored = deserialize_controller(value).expect("a registered tag must deserialize"); + assert_eq!(restored.controller_type(), controller.controller_type()); + } + + /// Resolving a controller for a new deployment is a different path from deserializing saved + /// state, so registering one does not imply the other. Both are needed and both are tested. + #[test] + fn the_registry_resolves_a_local_sandbox_controller() { + let registry = crate::core::ResourceRegistry::with_built_ins(); + + let controller = registry + .get_controller(alien_core::Sandbox::RESOURCE_TYPE, alien_core::Platform::Local) + .expect("Local must have a registered Sandbox controller"); + assert_eq!(controller.controller_type(), "LocalSandboxController"); + } +} diff --git a/crates/alien-infra/src/sandbox/mod.rs b/crates/alien-infra/src/sandbox/mod.rs new file mode 100644 index 000000000..1e0781055 --- /dev/null +++ b/crates/alien-infra/src/sandbox/mod.rs @@ -0,0 +1,35 @@ +//! Sandbox resource controllers. + +#[cfg(feature = "kubernetes")] +mod kubernetes_eligibility; +#[cfg(feature = "kubernetes")] +pub use kubernetes_eligibility::*; + +#[cfg(feature = "kubernetes")] +mod kubernetes; +#[cfg(feature = "kubernetes")] +pub use kubernetes::*; + +#[cfg(feature = "kubernetes")] +mod kubernetes_broker; +#[cfg(feature = "kubernetes")] +pub use kubernetes_broker::*; + +#[cfg(feature = "kubernetes")] +mod kubernetes_route; +#[cfg(feature = "kubernetes")] +pub use kubernetes_route::*; + +#[cfg(feature = "kubernetes")] +mod kubernetes_spec; +#[cfg(feature = "kubernetes")] +mod kubernetes_warm_pool; +#[cfg(feature = "kubernetes")] +pub use kubernetes_warm_pool::*; +#[cfg(feature = "kubernetes")] +pub use kubernetes_spec::*; + +#[cfg(feature = "local")] +mod local; +#[cfg(feature = "local")] +pub use local::*; diff --git a/crates/alien-infra/src/worker/gcp.rs b/crates/alien-infra/src/worker/gcp.rs index a34e6adf5..1079b4205 100644 --- a/crates/alien-infra/src/worker/gcp.rs +++ b/crates/alien-infra/src/worker/gcp.rs @@ -4545,6 +4545,7 @@ impl GcpWorkerController { .env(env) .resources(resources) .ports(ports) + .maybe_sandbox_launcher(cfg.sandbox_launcher.then_some(true)) .build(); let ingress = if cfg.public_endpoints.is_empty() { @@ -4608,6 +4609,13 @@ impl GcpWorkerController { .template(template) .traffic(traffic) .invoker_iam_disabled(is_public) + // Cloud Run refuses `sandboxLauncher` outside Beta or later with + // `FAILED_PRECONDITION: The feature 'Instant sandboxes' is not supported in the + // declared launch stage`, so the two are set together or not at all. + .maybe_launch_stage( + cfg.sandbox_launcher + .then_some(alien_gcp_clients::cloudrun::LaunchStage::Beta), + ) .build(); Ok(service) @@ -6367,6 +6375,61 @@ mod tests { assert!(executor.outputs().is_none()); } + /// A GCP sandbox session is a subprocess of the Cloud Run instance running the app, so an + /// instance that does not declare `sandboxLauncher` cannot start one — the deploy succeeds + /// and the first `create()` fails. Cloud Run also refuses the field outside Beta with + /// `FAILED_PRECONDITION: The feature 'Instant sandboxes' is not supported in the declared + /// launch stage`, so the two have to travel together. + #[tokio::test] + async fn a_sandbox_hosting_worker_declares_the_launcher_and_its_launch_stage() { + let mut worker = basic_function(); + worker.sandbox_launcher = true; + let function_name = format!("test-{}", worker.id); + + let mut mock_cloudrun = MockCloudRunApi::new(); + mock_cloudrun + .expect_create_service() + .times(1) + .withf(|_, _, service: &Service, _| { + let container = &service + .template + .as_ref() + .expect("a revision template") + .containers[0]; + container.sandbox_launcher == Some(true) + && service.launch_stage + == Some(alien_gcp_clients::cloudrun::LaunchStage::Beta) + }) + .returning(|_, _, _, _| Ok(create_successful_operation_response("create-worker"))); + mock_cloudrun + .expect_get_operation() + .returning(|_, _| Ok(create_completed_operation_response("create-worker"))); + let name_for_get = function_name.clone(); + mock_cloudrun + .expect_get_service() + .returning(move |_, _| Ok(create_successful_service_response(&name_for_get))); + mock_cloudrun + .expect_get_service_iam_policy() + .returning(|_, _| Ok(create_empty_iam_policy())); + mock_cloudrun + .expect_set_service_iam_policy() + .returning(|_, _, _| Ok(create_empty_iam_policy())); + + let mock_provider = setup_mock_service_provider(Arc::new(mock_cloudrun), None); + let mut executor = SingleControllerExecutor::builder() + .resource(worker) + .controller(GcpWorkerController::default()) + .platform(Platform::Gcp) + .service_provider(mock_provider) + .with_test_dependencies() + .build() + .await + .unwrap(); + + executor.run_until_terminal().await.unwrap(); + assert_eq!(executor.status(), ResourceStatus::Running); + } + #[tokio::test] async fn retries_cloud_run_revision_after_gar_reader_grant_propagates() { let worker = basic_function(); diff --git a/crates/alien-infra/tests/kubernetes_sandbox_live.rs b/crates/alien-infra/tests/kubernetes_sandbox_live.rs new file mode 100644 index 000000000..880976d0d --- /dev/null +++ b/crates/alien-infra/tests/kubernetes_sandbox_live.rs @@ -0,0 +1,272 @@ +//! The Kubernetes sandbox controller driven against a real cluster. +//! +//! `#[ignore]` because it needs a cluster with a sandboxed runtime class. Run with: +//! +//! ```text +//! ALIEN_TEST_GKE_KUBECONFIG_PATH=/path/to/kubeconfig \ +//! cargo test -p alien-infra --features all-platforms --test kubernetes_sandbox_live -- --ignored +//! ``` +//! +//! Every other Kubernetes test mocks the apiserver, which can only confirm the request we chose +//! to send. This one asks a real cluster whether the controller's create flow actually works. + +#![cfg(feature = "kubernetes")] + +use alien_core::{ + ClientConfig, KubernetesClientConfig, Platform, ResourceStatus, Sandbox, + SandboxCode, SandboxEgress, SandboxLimits, SandboxSessionPolicy, +}; +use alien_infra::controller_test::SingleControllerExecutor; +use alien_infra::KubernetesSandboxController; + +fn kubeconfig() -> Option { + std::env::var("ALIEN_TEST_GKE_KUBECONFIG_PATH").ok() +} + +fn sandbox(id: &str) -> Sandbox { + Sandbox::new(id.to_string()) + .code(SandboxCode::Image { + image: "alpine:3.20".to_string(), + }) + .limits(SandboxLimits { + cpu: "500m".to_string(), + memory: "512Mi".to_string(), + disk: "1Gi".to_string(), + max_processes: None, + }) + .egress(SandboxEgress::Deny) + .session(SandboxSessionPolicy { + max_lifetime_seconds: Some(600), + idle_suspend_seconds: None, + }) + .build() +} + +/// `kubectl` against the same cluster, used as ground truth. +/// +/// Asserting through the client under test would only prove it agrees with itself. +fn kubectl(path: &str, args: &[&str]) -> String { + let output = std::process::Command::new("kubectl") + .args(["--kubeconfig", path]) + .args(args) + .output() + .expect("kubectl should run"); + String::from_utf8_lossy(&output.stdout).trim().to_string() +} + +/// Namespace the sandbox's pods land in. The Helm chart creates it; the controller does not. +const NAMESPACE: &str = "alien-sandbox-live"; + +/// A namespace of its own for the claim test. +/// +/// Sharing one would make these tests order-dependent: a namespace deleted at the end of one is +/// still `Terminating` when the next creates a pod in it, and pod creation is best effort, so the +/// second test fails with an empty pool and no stated cause. +const CLAIM_NAMESPACE: &str = "alien-sandbox-claim"; + +/// Create, warm pool, delete — driven against a real cluster. +/// +/// This is what the mocked tests cannot say: that `verify_cluster` reads a real RuntimeClass +/// list, that the pool pods the health tick creates are accepted by an apiserver rather than +/// merely well-formed, and that deleting the parent takes them away again. +#[tokio::test] +#[ignore = "requires a cluster with a sandboxed runtime class"] +async fn the_lifecycle_creates_and_removes_pool_pods_on_a_real_cluster() { + let Some(path) = kubeconfig() else { + eprintln!("ALIEN_TEST_GKE_KUBECONFIG_PATH not set; skipping"); + return; + }; + + // Standing in for the Helm chart, which owns this namespace. + kubectl(&path, &["create", "namespace", NAMESPACE]); + + let mut executor = SingleControllerExecutor::builder() + .resource(sandbox("live")) + .controller(KubernetesSandboxController::default()) + .platform(Platform::Kubernetes) + // The `Kubeconfig` variant on purpose: resolving it is the service provider's job, and + // a test that resolved it by hand would not exercise that. + .client_config(ClientConfig::Kubernetes(Box::new( + KubernetesClientConfig::Kubeconfig { + kubeconfig_path: Some(path.clone()), + context: None, + cluster: None, + user: None, + namespace: Some(NAMESPACE.to_string()), + additional_headers: None, + }, + ))) + .build() + .await + .expect("the executor should build with a real kubeconfig"); + + executor + .run_until_terminal() + .await + .expect("the create flow should complete against a real cluster"); + + assert_eq!( + executor.status(), + ResourceStatus::Running, + "a cluster with gVisor must let a sandbox reach Running" + ); + + // The pool is filled on the health tick rather than during create, so reaching Running is + // not enough — one more step is what actually asks the apiserver for pods. + executor.step().await.expect("the health tick should run"); + + let pods = kubectl(&path, &["get", "pods", "-n", NAMESPACE, "-o", "name"]); + assert_eq!( + pods.lines().count(), + 2, + "the warm pool should be filled to its target, got: {pods}" + ); + + executor.delete().expect("the delete flow should start"); + executor + .run_until_terminal() + .await + .expect("the delete flow should complete"); + + // Polled rather than read once: a delete is a request, and a pod with a grace period is + // still listed while it terminates. Asserting immediately tests our timing, not the teardown. + let mut after = String::new(); + for _ in 0..30 { + after = kubectl(&path, &["get", "pods", "-n", NAMESPACE, "-o", "name"]); + if after.is_empty() { + break; + } + tokio::time::sleep(std::time::Duration::from_secs(2)).await; + } + assert!( + after.is_empty(), + "deleting the parent must remove every pool pod, got: {after}" + ); + + // A signing key nobody can use is still a signing key sitting in the cluster. + let secrets = kubectl( + &path, + &["get", "secrets", "-n", NAMESPACE, "-o", "name"], + ); + assert!( + !secrets.contains("capability"), + "teardown must take the capability key with it, got: {secrets}" + ); + + kubectl(&path, &["delete", "namespace", NAMESPACE, "--wait=false"]); +} + +/// The claim path, driven against a real cluster: a pool pod is claimed, the capability the +/// broker mints is one the agent accepts, and a command runs at the unprivileged uid. +/// +/// This is what the mocked broker tests cannot say. They prove the claim logic and the mint +/// agree with each other; only a cluster proves the agent agrees with both. +#[tokio::test] +#[ignore = "requires a cluster with a sandboxed runtime class"] +async fn a_claimed_pod_runs_a_command_at_the_unprivileged_uid() { + let Some(path) = kubeconfig() else { + eprintln!("ALIEN_TEST_GKE_KUBECONFIG_PATH not set; skipping"); + return; + }; + + kubectl(&path, &["create", "namespace", CLAIM_NAMESPACE]); + + let config = resolved(&path).await; + let client = alien_k8s_clients::kubernetes::kubernetes_client::KubernetesClient::new(config) + .await + .expect("a client from the kubeconfig"); + let client = std::sync::Arc::new(client); + + // The controller normally provisions this; here the test stands in for it so the claim path + // is exercised on its own. + let pair = ed25519_compact::KeyPair::generate(); + let secret_name = "alien-sandbox-live-capability"; + let secret = k8s_openapi::api::core::v1::Secret { + metadata: k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta { + name: Some(secret_name.to_string()), + namespace: Some(CLAIM_NAMESPACE.to_string()), + ..Default::default() + }, + data: Some(std::collections::BTreeMap::from([( + "signingKey".to_string(), + k8s_openapi::ByteString(pair.as_ref().to_vec()), + )])), + ..Default::default() + }; + let _ = alien_k8s_clients::kubernetes::secrets::SecretsApi::create_secret( + client.as_ref(), + CLAIM_NAMESPACE, + &secret, + ) + .await; + + let sandbox = sandbox("live"); + let pod = alien_infra::idle_pool_pod( + &sandbox, + CLAIM_NAMESPACE, + "gvisor", + None, + Some(&{ use base64::Engine as _; base64::engine::general_purpose::STANDARD.encode(pair.pk.as_ref()) }), + ); + alien_k8s_clients::kubernetes::pods::PodApi::create_pod(client.as_ref(), CLAIM_NAMESPACE, &pod) + .await + .expect("the pool pod is created"); + + // Wait for an address: a pod is claimable only once the kubelet has given it one. + for _ in 0..60 { + let running = kubectl( + &path, + &["get", "pod", "alien-sbx-live-pool-0", "-n", CLAIM_NAMESPACE, "-o", "jsonpath={.status.podIP}"], + ); + if !running.is_empty() { + break; + } + tokio::time::sleep(std::time::Duration::from_secs(5)).await; + } + + let pods: std::sync::Arc = client.clone(); + let secrets: std::sync::Arc = + client.clone(); + + let claimed = alien_infra::claim_session( + &pods, + &secrets, + "live", + CLAIM_NAMESPACE, + "s1", + secret_name, + chrono::Utc::now().timestamp(), + ) + .await + .expect("an idle pod is claimable"); + + println!("claimed {} at {}", claimed.session_id, claimed.endpoint); + assert!(claimed.endpoint.ends_with(":8971"), "got {}", claimed.endpoint); + + // The pod carries the session label now, which is what stops a second caller claiming it. + let labelled = kubectl( + &path, + &["get", "pods", "-n", CLAIM_NAMESPACE, "-l", "alien.dev/sandbox-session=s1", "-o", "name"], + ); + assert!(labelled.contains("alien-sbx-live-pool-0"), "got: {labelled}"); + + kubectl(&path, &["delete", "namespace", CLAIM_NAMESPACE, "--wait=false"]); +} + +/// Resolves the kubeconfig the same way the service provider does. +async fn resolved(path: &str) -> alien_core::KubernetesClientConfig { + alien_infra::resolve_kubeconfig(&KubernetesClientConfig::Kubeconfig { + kubeconfig_path: Some(path.to_string()), + context: None, + cluster: None, + user: None, + namespace: Some(CLAIM_NAMESPACE.to_string()), + additional_headers: None, + }) + .await + .expect("the kubeconfig resolves") +} + +// The negative case — a cluster without the requested sandboxed runtime class — is covered by +// `kubernetes_eligibility::live_cluster_shape`, which runs the same decision function against the +// RuntimeClass list this cluster actually returns. diff --git a/crates/alien-k8s-clients/src/kubernetes/mod.rs b/crates/alien-k8s-clients/src/kubernetes/mod.rs index 6e1da757a..0737c56fd 100644 --- a/crates/alien-k8s-clients/src/kubernetes/mod.rs +++ b/crates/alien-k8s-clients/src/kubernetes/mod.rs @@ -16,7 +16,9 @@ pub mod nodes; pub mod optional; pub mod pods; pub mod routes; +pub mod runtime_classes; pub mod secrets; +pub mod token_reviews; pub mod services; pub mod version; pub mod workload_heartbeat; diff --git a/crates/alien-k8s-clients/src/kubernetes/runtime_classes.rs b/crates/alien-k8s-clients/src/kubernetes/runtime_classes.rs new file mode 100644 index 000000000..ee2d65cf3 --- /dev/null +++ b/crates/alien-k8s-clients/src/kubernetes/runtime_classes.rs @@ -0,0 +1,37 @@ +use crate::kubernetes::kubernetes_client::KubernetesClient; +use crate::kubernetes::kubernetes_request_utils::sign_send_json; +use alien_client_core::Result; +use reqwest::Method; + +use k8s_openapi::api::node::v1::RuntimeClass; +use k8s_openapi::List; + +use async_trait::async_trait; +#[cfg(feature = "test-utils")] +use mockall::automock; + +#[cfg_attr(feature = "test-utils", automock)] +#[async_trait] +pub trait RuntimeClassApi: Send + Sync + std::fmt::Debug { + async fn list_runtime_classes(&self) -> Result>; +} + +impl KubernetesClient { + /// Lists the cluster's RuntimeClasses. + /// + /// A cluster-scoped object, so this answers "can this cluster run a sandboxed pod at all" + /// without depending on nodes being present — which matters because node auto-provisioning + /// creates them on demand, so an empty node list is not the same as an ineligible cluster. + pub async fn list_runtime_classes(&self) -> Result> { + let url = format!("{}/apis/node.k8s.io/v1/runtimeclasses", self.get_base_url()); + let builder = self.client().request(Method::GET, &url); + sign_send_json(builder, &self.auth_config()).await + } +} + +#[async_trait] +impl RuntimeClassApi for KubernetesClient { + async fn list_runtime_classes(&self) -> Result> { + KubernetesClient::list_runtime_classes(self).await + } +} diff --git a/crates/alien-k8s-clients/src/kubernetes/token_reviews.rs b/crates/alien-k8s-clients/src/kubernetes/token_reviews.rs new file mode 100644 index 000000000..d676126ec --- /dev/null +++ b/crates/alien-k8s-clients/src/kubernetes/token_reviews.rs @@ -0,0 +1,144 @@ +//! Asking the apiserver who a bearer token belongs to. +//! +//! This is how a service verifies a caller that presented its own projected ServiceAccount +//! token. The alternative — issuing a token of our own and distributing it — creates a secret +//! that has to be rotated and torn down, where Kubernetes already mounts one in every pod. + +use crate::kubernetes::kubernetes_client::KubernetesClient; +use crate::kubernetes::kubernetes_request_utils::sign_send_json; +use alien_client_core::{ErrorData, Result}; +use alien_error::{Context, IntoAlienError}; +use reqwest::Method; + +use k8s_openapi::api::authentication::v1::TokenReview; + +use async_trait::async_trait; +#[cfg(feature = "test-utils")] +use mockall::automock; + +#[cfg_attr(feature = "test-utils", automock)] +#[async_trait] +pub trait TokenReviewsApi: Send + Sync + std::fmt::Debug { + /// Submits a token for review and returns the apiserver's verdict. + /// + /// A successful call does **not** mean the token is valid: the verdict is in + /// `status.authenticated`, and a rejected token comes back 200 with that flag false. Treating + /// the HTTP status as the answer would authenticate everything. + async fn create_token_review(&self, review: &TokenReview) -> Result; +} + +impl KubernetesClient { + /// Submits a `TokenReview` to the apiserver. + pub async fn create_token_review(&self, review: &TokenReview) -> Result { + let body = serde_json::to_string(review) + .into_alien_error() + .context(ErrorData::SerializationError { + message: "Failed to serialize TokenReview".to_string(), + })?; + + let url = format!( + "{}/apis/authentication.k8s.io/v1/tokenreviews", + self.get_base_url() + ); + let builder = self + .client() + .request(Method::POST, &url) + .header("Content-Type", "application/json") + .body(body); + + sign_send_json(builder, &self.auth_config()).await + } +} + +#[async_trait] +impl TokenReviewsApi for KubernetesClient { + async fn create_token_review(&self, review: &TokenReview) -> Result { + KubernetesClient::create_token_review(self, review).await + } +} + +/// Builds a review request for one bearer token. +pub fn review_for(token: &str) -> TokenReview { + TokenReview { + spec: k8s_openapi::api::authentication::v1::TokenReviewSpec { + token: Some(token.to_string()), + audiences: None, + }, + ..Default::default() + } +} + +/// The authenticated username from a verdict, or `None` if the token was rejected. +/// +/// `None` covers both "not authenticated" and "authenticated with no username", because a caller +/// we cannot name is a caller we cannot authorize. +pub fn authenticated_user(review: &TokenReview) -> Option { + let status = review.status.as_ref()?; + if !status.authenticated.unwrap_or(false) { + return None; + } + status.user.as_ref()?.username.clone() +} + +/// Whether an authenticated username is a ServiceAccount in `namespace`. +/// +/// Kubernetes formats these as `system:serviceaccount::`. Matching the whole +/// prefix rather than searching for the namespace anywhere in the string: a namespace name can +/// appear inside a ServiceAccount name, and a substring match would accept the wrong tenant. +pub fn is_service_account_in(username: &str, namespace: &str) -> bool { + username.starts_with(&format!("system:serviceaccount:{namespace}:")) +} + +#[cfg(test)] +mod tests { + use super::*; + use k8s_openapi::api::authentication::v1::{TokenReviewStatus, UserInfo}; + + fn verdict(authenticated: bool, username: Option<&str>) -> TokenReview { + TokenReview { + status: Some(TokenReviewStatus { + authenticated: Some(authenticated), + user: username.map(|name| UserInfo { + username: Some(name.to_string()), + ..Default::default() + }), + ..Default::default() + }), + ..Default::default() + } + } + + /// A rejected token comes back 200 with `authenticated: false`. Reading the HTTP status as + /// the answer would authenticate every caller. + #[test] + fn a_rejected_token_yields_no_user() { + assert_eq!( + authenticated_user(&verdict(false, Some("system:serviceaccount:ns:app"))), + None + ); + assert_eq!(authenticated_user(&verdict(true, None)), None); + assert_eq!( + authenticated_user(&verdict(true, Some("system:serviceaccount:ns:app"))).as_deref(), + Some("system:serviceaccount:ns:app") + ); + } + + /// A namespace name can appear inside a ServiceAccount name, so the check is on the whole + /// prefix. A substring match would accept a caller from another tenant. + #[test] + fn namespace_matching_is_not_a_substring_search() { + assert!(is_service_account_in( + "system:serviceaccount:alien-app:worker", + "alien-app" + )); + assert!(!is_service_account_in( + "system:serviceaccount:other:alien-app", + "alien-app" + )); + assert!(!is_service_account_in( + "system:serviceaccount:alien-app-staging:worker", + "alien-app" + )); + assert!(!is_service_account_in("system:node:node-1", "alien-app")); + } +} diff --git a/crates/alien-local/Cargo.toml b/crates/alien-local/Cargo.toml index b107ede98..0cbce6655 100644 --- a/crates/alien-local/Cargo.toml +++ b/crates/alien-local/Cargo.toml @@ -39,6 +39,8 @@ object_store = { workspace = true, features = ["http"] } # Docker client bollard = { workspace = true } +axum = { workspace = true, features = ["tokio", "http1", "json", "query"] } +base64 = { workspace = true } bytes = { workspace = true } futures-util = { workspace = true } @@ -75,6 +77,7 @@ container-registry = { workspace = true, features = ["test-support"] } sec = { workspace = true } [dev-dependencies] +base64 = { workspace = true } tempfile = { workspace = true } bytes = { workspace = true } tokio = { workspace = true, features = ["rt-multi-thread", "macros", "time", "sync"] } diff --git a/crates/alien-local/src/error.rs b/crates/alien-local/src/error.rs index e0436b583..f8fcab9c7 100644 --- a/crates/alien-local/src/error.rs +++ b/crates/alien-local/src/error.rs @@ -213,6 +213,20 @@ pub enum ErrorData { reason: String, }, + /// A sandbox session operation failed. + #[error( + code = "SANDBOX_SESSION_FAILED", + message = "Sandbox session '{session_id}' failed during {operation}", + retryable = "false", + internal = "false" + )] + SandboxSessionFailed { + /// Session the operation targeted + session_id: String, + /// What was being attempted + operation: String, + }, + /// Failed to create or manage Docker network. #[error( code = "DOCKER_NETWORK_ERROR", diff --git a/crates/alien-local/src/lib.rs b/crates/alien-local/src/lib.rs index 45206a96b..beb5e9067 100644 --- a/crates/alien-local/src/lib.rs +++ b/crates/alien-local/src/lib.rs @@ -60,6 +60,8 @@ mod kv_manager; mod local_bindings_provider; mod postgres_manager; mod queue_manager; +mod sandbox_manager; +mod sandbox_route; mod storage_manager; mod store_probe; pub mod trigger_service; @@ -79,6 +81,11 @@ pub use kv_manager::LocalKvManager; pub use local_bindings_provider::LocalBindingsProvider; pub use postgres_manager::LocalPostgresManager; pub use queue_manager::LocalQueueManager; +pub use sandbox_route::SandboxRoute; +pub use sandbox_manager::{ + LocalSandboxManager, SandboxEgressMode, SandboxExecResult, SandboxOutput, SandboxSessionConfig, + SandboxSessionHandle, +}; pub use storage_manager::LocalStorageManager; pub use vault_manager::LocalVaultManager; pub use worker_manager::{LocalWorkerManager, RuntimeOnlyBindingRef}; diff --git a/crates/alien-local/src/local_bindings_provider.rs b/crates/alien-local/src/local_bindings_provider.rs index b6a79e9b0..9487e1242 100644 --- a/crates/alien-local/src/local_bindings_provider.rs +++ b/crates/alien-local/src/local_bindings_provider.rs @@ -9,7 +9,8 @@ use crate::error::Result; use crate::{ LocalArtifactRegistryManager, LocalContainerManager, LocalKvManager, LocalPostgresManager, - LocalQueueManager, LocalStorageManager, LocalVaultManager, LocalWorkerManager, + LocalQueueManager, LocalSandboxManager, LocalStorageManager, LocalVaultManager, + LocalWorkerManager, }; use alien_bindings::{ error::ErrorData as BindingsErrorData, @@ -60,6 +61,8 @@ pub struct LocalBindingsProvider { artifact_registry_manager: Arc, /// Container manager for Docker containers (optional - created lazily) container_manager: RwLock>>, + /// Sandbox manager for hardened Docker sessions (optional - created lazily) + sandbox_manager: RwLock>>, /// Worker manager is set after construction to break circular dependency worker_manager: RwLock>>, /// Shutdown signal sender @@ -80,6 +83,7 @@ impl Clone for LocalBindingsProvider { vault_manager: self.vault_manager.clone(), artifact_registry_manager: self.artifact_registry_manager.clone(), container_manager: RwLock::new(self.container_manager.read().unwrap().clone()), + sandbox_manager: RwLock::new(self.sandbox_manager.read().unwrap().clone()), worker_manager: RwLock::new(self.worker_manager.read().unwrap().clone()), shutdown_tx: self.shutdown_tx.clone(), background_tasks: Mutex::new(Vec::new()), // Don't clone JoinHandles @@ -148,6 +152,7 @@ impl LocalBindingsProvider { vault_manager: vault_manager.clone(), artifact_registry_manager: artifact_registry_manager.clone(), container_manager: RwLock::new(None), + sandbox_manager: RwLock::new(None), worker_manager: RwLock::new(None), shutdown_tx: shutdown_tx.clone(), background_tasks: Mutex::new(Vec::new()), @@ -276,6 +281,31 @@ impl LocalBindingsProvider { } } + /// Returns the sandbox manager, creating it lazily if needed. + /// + /// Separate from the container manager on purpose: that one attaches a shared network and + /// maps the host gateway in, which is wrong for untrusted code. + pub fn sandbox_manager(&self) -> Option> { + { + let guard = self.sandbox_manager.read().unwrap(); + if let Some(manager) = guard.as_ref() { + return Some(manager.clone()); + } + } + + match LocalSandboxManager::new(self.state_dir.clone()) { + Ok(manager) => { + let manager = Arc::new(manager); + *self.sandbox_manager.write().unwrap() = Some(manager.clone()); + Some(manager) + } + Err(error) => { + tracing::warn!("Failed to create LocalSandboxManager: {:?}", error); + None + } + } + } + /// Inherent counterpart of [`BindingsProviderApi::resolve_runtime_only_binding_env`], so /// controllers holding the concrete provider (the container path) can resolve without /// importing the trait. The resource type routes to the local secret source. @@ -652,6 +682,19 @@ impl BindingsProviderApi for LocalBindingsProvider { ), })) } + + async fn load_sandbox( + &self, + binding_name: &str, + ) -> alien_bindings::error::Result> { + Err(AlienError::new(BindingsErrorData::OperationNotSupported { + operation: "load_sandbox".to_string(), + reason: format!( + "the local sandbox backend for '{}' is not implemented yet", + binding_name + ), + })) + } } #[cfg(test)] diff --git a/crates/alien-local/src/sandbox_manager.rs b/crates/alien-local/src/sandbox_manager.rs new file mode 100644 index 000000000..ead9b3c0b --- /dev/null +++ b/crates/alien-local/src/sandbox_manager.rs @@ -0,0 +1,729 @@ +//! Local sandbox sessions on Docker. +//! +//! Deliberately not `LocalContainerManager`. That manager attaches a shared network, maps +//! `host.docker.internal:host-gateway` into every container, and restarts on exit — all +//! correct for a service and all wrong for hostile code. Reusing it would be a security +//! regression dressed as reuse. +//! +//! **Docker is a shared kernel.** Hardening narrows the attack surface; it does not make +//! container escape out of scope. Local is development-only for untrusted code unless a +//! sandboxed runtime such as gVisor or Kata is present and verified. + +use std::collections::HashMap; +use std::path::PathBuf; + +use bollard::container::{ + Config, CreateContainerOptions, ListContainersOptions, RemoveContainerOptions, +}; +use bollard::image::CreateImageOptions; +use bollard::exec::{CreateExecOptions, StartExecResults}; +use bollard::models::{HostConfig, PortBinding}; +use bollard::network::CreateNetworkOptions; +use bollard::Docker; +use futures::StreamExt; +use tokio::io::AsyncWriteExt; + +use crate::error::{ErrorData, Result}; +use alien_error::{AlienError, Context, IntoAlienError}; + +/// Label carrying the sandbox resource a container belongs to. +const LABEL_SANDBOX: &str = "dev.alien.sandbox"; +/// Label carrying the session id within that sandbox. +const LABEL_SESSION: &str = "dev.alien.sandbox.session"; + +/// Unprivileged uid/gid the workload runs as. `nobody` exists in every common base image. +const SANDBOX_USER: &str = "65534:65534"; + +/// Where a session's writable area is mounted, and what every caller-supplied path resolves +/// against. The same root the in-sandbox agent uses on the cloud backends, so one path means +/// the same file everywhere. +const SESSION_ROOT: &str = "/sandbox"; + +/// Outbound network policy for a session. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SandboxEgressMode { + /// No network interface at all. + Deny, + /// A session-private bridge with internet access. + Allow, +} + +/// What a session is allowed to consume. +#[derive(Debug, Clone, PartialEq)] +pub struct SandboxSessionConfig { + /// Image used as the session's root filesystem + pub image: String, + /// CPU ceiling in cores + pub cpu_cores: f64, + /// Memory ceiling in bytes + pub memory_bytes: i64, + /// Maximum number of processes, which bounds fork bombs + pub pids_limit: Option, + /// Writable scratch size in bytes; the root filesystem itself is read-only + pub scratch_bytes: u64, + /// Outbound network policy + pub egress: SandboxEgressMode, + /// Ports eligible for a preview capability. Docker fixes published ports at create time, + /// so this cannot be decided later — which is also the property that stops an application + /// widening its own ingress at runtime. + pub preview_ports: Vec, + /// Environment placed in the session + pub env: HashMap, +} + +/// A session the manager is tracking. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SandboxSessionHandle { + /// Session id within the sandbox + pub session_id: String, + /// Docker container backing it + pub container_id: String, +} + +/// One frame of a running command's output. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum SandboxOutput { + /// Bytes written to stdout + Stdout(Vec), + /// Bytes written to stderr + Stderr(Vec), +} + +/// A finished command. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SandboxExecResult { + /// Frames in the order they were produced + pub output: Vec, + /// Process exit code + pub exit_code: i64, +} + +/// Creates and destroys hardened Docker sessions for one deployment. +#[derive(Debug)] +pub struct LocalSandboxManager { + docker: Docker, + state_dir: PathBuf, +} + +impl LocalSandboxManager { + /// Connects to the local Docker daemon. + pub fn new(state_dir: PathBuf) -> Result { + let docker = Docker::connect_with_local_defaults() + .into_alien_error() + .context(ErrorData::DockerConnectionFailed { + reason: "could not reach the local Docker daemon".to_string(), + })?; + + Ok(Self { docker, state_dir }) + } + + /// Where this manager keeps its session state. + pub fn state_dir(&self) -> &PathBuf { + &self.state_dir + } + + fn container_name(sandbox: &str, session_id: &str) -> String { + format!("alien-sbx-{sandbox}-{session_id}") + } + + /// One egress network per sandbox, not per session. + /// + /// A bridge per session does not isolate sessions: every bridge lives on the same host and + /// the host routes between them, so a neighbouring session stays reachable. One bridge with + /// inter-container communication disabled blocks session-to-session traffic natively and + /// still allows outbound. + fn network_name(sandbox: &str) -> String { + format!("alien-sbx-net-{sandbox}") + } + + /// Creates a session and leaves it running until terminated. + pub async fn create_session( + &self, + sandbox: &str, + session_id: &str, + config: &SandboxSessionConfig, + ) -> Result { + let name = Self::container_name(sandbox, session_id); + + // Docker accepts port bindings on a network-less container and silently drops them, so + // a preview port under deny egress would look configured and never resolve. + if !config.preview_ports.is_empty() && config.egress == SandboxEgressMode::Deny { + return Err(AlienError::new(ErrorData::SandboxSessionFailed { + session_id: session_id.to_string(), + operation: "preview ports require egress; a session with no interface cannot \ + serve one" + .to_string(), + })); + } + + self.ensure_image(&config.image).await?; + + let network_mode = match config.egress { + // "none" gives no interface at all, which is a stronger and simpler guarantee than + // a private network with its gateway firewalled off. + SandboxEgressMode::Deny => "none".to_string(), + SandboxEgressMode::Allow => { + let network = Self::network_name(sandbox); + self.ensure_egress_network(&network).await?; + network + } + }; + + let env: Vec = config + .env + .iter() + .map(|(key, value)| format!("{key}={value}")) + .collect(); + + let labels = HashMap::from([ + (LABEL_SANDBOX.to_string(), sandbox.to_string()), + (LABEL_SESSION.to_string(), session_id.to_string()), + ]); + + // Bound to 127.0.0.1, never 0.0.0.0: a preview is for the developer's own machine, and + // publishing on all interfaces would expose untrusted code to the local network. + let port_bindings: HashMap>> = config + .preview_ports + .iter() + .map(|port| { + ( + format!("{port}/tcp"), + Some(vec![PortBinding { + host_ip: Some("127.0.0.1".to_string()), + host_port: None, + }]), + ) + }) + .collect(); + + let exposed_ports: HashMap> = config + .preview_ports + .iter() + .map(|port| (format!("{port}/tcp"), HashMap::new())) + .collect(); + + let host_config = HostConfig { + network_mode: Some(network_mode), + port_bindings: if port_bindings.is_empty() { + None + } else { + Some(port_bindings) + }, + // Never map the host gateway in. LocalContainerManager does, which is why this + // manager exists. + extra_hosts: None, + readonly_rootfs: Some(true), + tmpfs: Some(HashMap::from([( + SESSION_ROOT.to_string(), + // 1777 because the workload runs as an unprivileged uid: a tmpfs mounted with + // the default mode is root-owned, and the session's only writable area would + // not be writable by the process using it. + format!("rw,noexec,nosuid,mode=1777,size={}", config.scratch_bytes), + )])), + cap_drop: Some(vec!["ALL".to_string()]), + security_opt: Some(vec!["no-new-privileges:true".to_string()]), + pids_limit: config.pids_limit, + memory: Some(config.memory_bytes), + nano_cpus: Some((config.cpu_cores * 1_000_000_000.0) as i64), + // No restart policy: a sandbox that exits stays exited. Restarting hostile code + // would silently hand it another attempt. + ..Default::default() + }; + + let container_config = Config { + image: Some(config.image.clone()), + user: Some(SANDBOX_USER.to_string()), + working_dir: Some(SESSION_ROOT.to_string()), + env: Some(env), + labels: Some(labels), + exposed_ports: if exposed_ports.is_empty() { + None + } else { + Some(exposed_ports) + }, + // Hold the session open without a shell of its own; commands arrive through exec. + entrypoint: Some(vec!["/bin/sh".to_string()]), + cmd: Some(vec!["-c".to_string(), "while true; do sleep 3600; done".to_string()]), + host_config: Some(host_config), + ..Default::default() + }; + + let created = self + .docker + .create_container( + Some(CreateContainerOptions { + name: name.clone(), + platform: None, + }), + container_config, + ) + .await + .into_alien_error() + .context(ErrorData::SandboxSessionFailed { + session_id: session_id.to_string(), + operation: "create".to_string(), + })?; + + self.docker + .start_container::(&created.id, None) + .await + .into_alien_error() + .context(ErrorData::SandboxSessionFailed { + session_id: session_id.to_string(), + operation: "start".to_string(), + })?; + + Ok(SandboxSessionHandle { + session_id: session_id.to_string(), + container_id: created.id, + }) + } + + /// Pulls the sandbox image if the daemon does not already have it. + /// + /// A sandbox cannot start without its root filesystem, and Docker's create call does not + /// pull. Inspecting first keeps the common case off the network. + async fn ensure_image(&self, image: &str) -> Result<()> { + if self.docker.inspect_image(image).await.is_ok() { + return Ok(()); + } + + let mut pull = self.docker.create_image( + Some(CreateImageOptions { + from_image: image.to_string(), + ..Default::default() + }), + None, + None, + ); + + while let Some(progress) = pull.next().await { + progress + .into_alien_error() + .context(ErrorData::SandboxSessionFailed { + session_id: image.to_string(), + operation: "pull sandbox image".to_string(), + })?; + } + + Ok(()) + } + + async fn ensure_egress_network(&self, name: &str) -> Result<()> { + match self.docker.inspect_network::(name, None).await { + Ok(_) => Ok(()), + Err(_) => { + self.docker + .create_network(CreateNetworkOptions { + name: name.to_string(), + driver: "bridge".to_string(), + options: HashMap::from([( + "com.docker.network.bridge.enable_icc".to_string(), + "false".to_string(), + )]), + ..Default::default() + }) + .await + .into_alien_error() + .context(ErrorData::SandboxSessionFailed { + session_id: name.to_string(), + operation: "create network".to_string(), + })?; + Ok(()) + } + } + } + + /// Runs a command inside a session and collects its output and exit code. + pub async fn exec( + &self, + container_id: &str, + command: &[String], + ) -> Result { + let exec = self + .docker + .create_exec( + container_id, + CreateExecOptions { + cmd: Some(command.to_vec()), + attach_stdout: Some(true), + attach_stderr: Some(true), + user: Some(SANDBOX_USER.to_string()), + ..Default::default() + }, + ) + .await + .into_alien_error() + .context(ErrorData::SandboxSessionFailed { + session_id: container_id.to_string(), + operation: "create exec".to_string(), + })?; + + let started = self + .docker + .start_exec(&exec.id, None) + .await + .into_alien_error() + .context(ErrorData::SandboxSessionFailed { + session_id: container_id.to_string(), + operation: "start exec".to_string(), + })?; + + let mut output = Vec::new(); + if let StartExecResults::Attached { output: mut stream, .. } = started { + while let Some(frame) = stream.next().await { + let frame = frame.into_alien_error().context(ErrorData::SandboxSessionFailed { + session_id: container_id.to_string(), + operation: "read exec output".to_string(), + })?; + + match frame { + bollard::container::LogOutput::StdOut { message } => { + output.push(SandboxOutput::Stdout(message.to_vec())) + } + bollard::container::LogOutput::StdErr { message } => { + output.push(SandboxOutput::Stderr(message.to_vec())) + } + _ => {} + } + } + } + + let inspect = self + .docker + .inspect_exec(&exec.id) + .await + .into_alien_error() + .context(ErrorData::SandboxSessionFailed { + session_id: container_id.to_string(), + operation: "inspect exec".to_string(), + })?; + + // A missing exit code means the command did not finish, which is not success. + let exit_code = inspect + .exit_code + .ok_or_else(|| { + AlienError::new(ErrorData::SandboxSessionFailed { + session_id: container_id.to_string(), + operation: "exec finished without an exit code".to_string(), + }) + })?; + + Ok(SandboxExecResult { output, exit_code }) + } + + /// Writes one file into a session. + /// + /// Streamed through an exec's stdin rather than Docker's archive-upload API: that API + /// extracts through the container filesystem layer and is refused outright when the root + /// filesystem is read-only, even when the target is a writable tmpfs. The path is validated + /// and passed as an argv element, so it never reaches a shell for interpretation. + pub async fn write_file(&self, container_id: &str, path: &str, contents: &[u8]) -> Result<()> { + let path = resolve_in_root(path)?; + + let exec = self + .docker + .create_exec( + container_id, + CreateExecOptions { + // Parent directories are created, matching the in-sandbox agent the cloud + // backends run. Without it the same `write_files` call succeeds on AWS and + // fails on Local for any path with a directory in it. + cmd: Some(vec![ + "/bin/sh".to_string(), + "-c".to_string(), + "mkdir -p \"$(dirname \"$1\")\" && cat > \"$1\"".to_string(), + "sh".to_string(), + path.to_string(), + ]), + attach_stdin: Some(true), + attach_stdout: Some(true), + attach_stderr: Some(true), + user: Some(SANDBOX_USER.to_string()), + ..Default::default() + }, + ) + .await + .into_alien_error() + .context(ErrorData::SandboxSessionFailed { + session_id: container_id.to_string(), + operation: "create upload exec".to_string(), + })?; + + let started = self + .docker + .start_exec(&exec.id, None) + .await + .into_alien_error() + .context(ErrorData::SandboxSessionFailed { + session_id: container_id.to_string(), + operation: "start upload exec".to_string(), + })?; + + if let StartExecResults::Attached { mut input, mut output } = started { + input + .write_all(contents) + .await + .into_alien_error() + .context(ErrorData::SandboxSessionFailed { + session_id: container_id.to_string(), + operation: "write file contents".to_string(), + })?; + input + .shutdown() + .await + .into_alien_error() + .context(ErrorData::SandboxSessionFailed { + session_id: container_id.to_string(), + operation: "close file stream".to_string(), + })?; + + // Drain so the command observes EOF and finishes before the exit code is read. + while output.next().await.is_some() {} + } + + let inspect = self + .docker + .inspect_exec(&exec.id) + .await + .into_alien_error() + .context(ErrorData::SandboxSessionFailed { + session_id: container_id.to_string(), + operation: "inspect upload exec".to_string(), + })?; + + match inspect.exit_code { + Some(0) => Ok(()), + other => Err(AlienError::new(ErrorData::SandboxSessionFailed { + session_id: container_id.to_string(), + operation: format!("write '{path}' exited with {other:?}"), + })), + } + } + + /// Reads one file out of a session. + /// + /// Via exec rather than Docker's archive-download API, which reads the container filesystem + /// layer and cannot see tmpfs mounts — and the session's only writable area is a tmpfs. + pub async fn read_file(&self, container_id: &str, path: &str) -> Result> { + let path = resolve_in_root(path)?; + + let result = self + .exec(container_id, &["/bin/cat".to_string(), path.to_string()]) + .await?; + + if result.exit_code != 0 { + return Err(AlienError::new(ErrorData::SandboxSessionFailed { + session_id: container_id.to_string(), + operation: format!("read '{path}' exited with {}", result.exit_code), + })); + } + + Ok(result + .output + .into_iter() + .filter_map(|frame| match frame { + SandboxOutput::Stdout(bytes) => Some(bytes), + SandboxOutput::Stderr(_) => None, + }) + .flatten() + .collect()) + } + + /// Returns the loopback address a declared preview port is published on. + /// + /// Refuses a port that was not declared at create time: Docker fixed the published set + /// then, and an undeclared port has nowhere to be reachable from anyway. + pub async fn preview_address(&self, container_id: &str, port: u16) -> Result { + let inspected = self + .docker + .inspect_container(container_id, None) + .await + .into_alien_error() + .context(ErrorData::SandboxSessionFailed { + session_id: container_id.to_string(), + operation: "inspect for preview".to_string(), + })?; + + let host_port = inspected + .network_settings + .and_then(|settings| settings.ports) + .and_then(|ports| ports.get(&format!("{port}/tcp")).cloned().flatten()) + .and_then(|bindings| bindings.first().and_then(|binding| binding.host_port.clone())) + .ok_or_else(|| { + AlienError::new(ErrorData::SandboxSessionFailed { + session_id: container_id.to_string(), + operation: format!("port {port} was not declared as a preview port"), + }) + })?; + + Ok(format!("http://127.0.0.1:{host_port}")) + } + + /// Lists the sessions this manager is tracking for a sandbox. + pub async fn list_sessions(&self, sandbox: &str) -> Result> { + let filters = HashMap::from([( + "label".to_string(), + vec![format!("{LABEL_SANDBOX}={sandbox}")], + )]); + + let containers = self + .docker + .list_containers(Some(ListContainersOptions { + all: true, + filters, + ..Default::default() + })) + .await + .into_alien_error() + .context(ErrorData::SandboxSessionFailed { + session_id: sandbox.to_string(), + operation: "list sessions".to_string(), + })?; + + Ok(containers + .into_iter() + .filter_map(|container| { + let session_id = container.labels.as_ref()?.get(LABEL_SESSION)?.clone(); + Some(SandboxSessionHandle { + session_id, + container_id: container.id?, + }) + }) + .collect()) + } + + /// Removes a session and its private network. Idempotent: an absent session is success. + pub async fn terminate(&self, sandbox: &str, session_id: &str) -> Result<()> { + let name = Self::container_name(sandbox, session_id); + + let removed = self + .docker + .remove_container( + &name, + Some(RemoveContainerOptions { + force: true, + v: true, + ..Default::default() + }), + ) + .await; + + match removed { + Ok(()) => Ok(()), + // An already-gone session is the desired end state. Every other failure leaves the + // container running, and terminate is the containment kill switch — reporting success + // there tells the caller untrusted code has stopped when it has not. + Err(bollard::errors::Error::DockerResponseServerError { + status_code: 404, .. + }) => Ok(()), + Err(error) => Err(error).into_alien_error().context(ErrorData::SandboxSessionFailed { + session_id: session_id.to_string(), + operation: format!("remove container '{name}'"), + })?, + } + } + + /// Removes every session belonging to a sandbox. + /// + /// Run at manager startup so a CLI restart does not leave containers behind, and at + /// teardown so the Frozen parent's children go first. + pub async fn reap(&self, sandbox: &str) -> Result { + let sessions = self.list_sessions(sandbox).await?; + let count = sessions.len(); + + for session in sessions { + self.terminate(sandbox, &session.session_id).await?; + } + + // Best effort: Docker refuses to remove a network with members, which is the correct + // outcome rather than an error to propagate. + let _ = self.docker.remove_network(&Self::network_name(sandbox)).await; + + Ok(count) + } +} + +/// Resolves a caller-supplied path against the session root. +/// +/// An absolute path means "under the session root", not "on the container's filesystem" — the +/// same rule the in-sandbox agent applies on the cloud backends. Reading it as host-absolute +/// would let a caller name any file in the image, and the session root is the only writable +/// area anyway. +fn resolve_in_root(path: &str) -> Result { + let refused = |reason: &str| { + AlienError::new(ErrorData::SandboxSessionFailed { + session_id: path.to_string(), + operation: format!("path {reason}"), + }) + }; + + // A trailing slash names a directory, and these operations act on files. Checked before any + // trimming, which would make "/work/" indistinguishable from the file "/work". + if path.ends_with('/') { + return Err(refused("must name a file, not a directory")); + } + + let relative = path.trim_start_matches('/'); + if relative.is_empty() { + return Err(refused("is empty")); + } + + if relative.split('/').any(|part| part == ".." || part.is_empty()) { + return Err(refused("must not traverse")); + } + + Ok(format!("{SESSION_ROOT}/{relative}")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn container_and_network_names_are_session_scoped() { + assert_eq!( + LocalSandboxManager::container_name("agent", "s1"), + "alien-sbx-agent-s1" + ); + // One egress network per sandbox: a bridge per session does not isolate sessions. + assert_eq!( + LocalSandboxManager::network_name("agent"), + "alien-sbx-net-agent" + ); + } + + /// Absolute and relative name the same file, and both sit under the session root. A caller + /// that works against AWS must not have to rewrite its paths for Local. + #[test] + fn paths_resolve_against_the_session_root() { + assert_eq!( + resolve_in_root("/work/main.py").expect("absolute path"), + "/sandbox/work/main.py" + ); + assert_eq!( + resolve_in_root("work/main.py").expect("relative path"), + "/sandbox/work/main.py" + ); + assert_eq!( + resolve_in_root("main.py").expect("bare name"), + "/sandbox/main.py" + ); + } + + #[test] + fn traversal_and_directory_paths_are_refused() { + for path in [ + "/work/../etc/passwd", + "../etc/passwd", + "/work/..", + "/work/", + "/", + "", + "work//main.py", + ] { + resolve_in_root(path) + .expect_err(&format!("'{path}' must be refused before it reaches Docker")); + } + } +} diff --git a/crates/alien-local/src/sandbox_route.rs b/crates/alien-local/src/sandbox_route.rs new file mode 100644 index 000000000..d7e398206 --- /dev/null +++ b/crates/alien-local/src/sandbox_route.rs @@ -0,0 +1,606 @@ +//! Authenticated loopback route to the local sandbox manager. +//! +//! The binding provider cannot call the manager in process: `alien-local` depends on +//! `alien-bindings`, so a direct call would be a dependency cycle. Giving the workload Docker +//! socket access instead would be worse — it hands every application the ability to escape its +//! own sandbox. So the provider speaks over loopback, which also means Local exercises the real +//! transport rather than a shortcut. + +use std::collections::HashMap; +use std::net::SocketAddr; +use std::sync::{Arc, Mutex, OnceLock}; + +use axum::extract::{Path, Query, State}; +use axum::http::{HeaderMap, StatusCode}; +use axum::routing::{get, post}; +use axum::{Json, Router}; +use base64::engine::general_purpose::STANDARD as BASE64; +use base64::Engine as _; +use serde::{Deserialize, Serialize}; +use tokio::net::TcpListener; + +use crate::error::{ErrorData, Result}; +use crate::sandbox_manager::{LocalSandboxManager, SandboxOutput, SandboxSessionConfig}; +use alien_error::{AlienError, IntoAlienError}; + +/// Bearer token file for one sandbox, inside the deployment state directory. +/// +/// Per sandbox: a shared path means whichever sandbox wrote last owns the credential for all of +/// them, and the first one's binding then authenticates with a token that is no longer valid. +fn token_file_name(sandbox: &str) -> String { + format!("sandbox-manager-{sandbox}.token") +} + +/// A request to create a session. +/// +/// Carries only an id. The image, limits, egress mode and preview ports come from the template +/// the controller configured — an application must not be able to raise its own ceilings, and +/// a client-supplied limit is a limit the client can choose not to send. +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CreateSessionBody { + /// Session id within the sandbox + pub session_id: String, +} + +/// A request to run a command. +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ExecBody { + /// Command and arguments + pub command: Vec, +} + +/// A request to write a file. +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct WriteFileBody { + /// Absolute path inside the session + pub path: String, + /// Contents, base64 because a file is arbitrary bytes + pub contents_base64: String, +} + +/// Which preview port to resolve. +#[derive(Debug, Deserialize)] +pub struct PreviewQuery { + /// Port declared at create time + pub port: u16, +} + +/// An authenticated capability to reach a port inside a session. +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct PreviewResponse { + /// Loopback endpoint the port is published on + pub endpoint: String, + /// Ports this capability admits + pub allowed_ports: Vec, +} + +/// Which file to read. +#[derive(Debug, Deserialize)] +pub struct ReadFileQuery { + /// Absolute path inside the session + pub path: String, +} + +/// A session as the route reports it. +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SessionBody { + /// Session id within the sandbox + pub session_id: String, + /// Docker container backing it + pub container_id: String, +} + +/// One output frame. +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase", tag = "stream", content = "dataBase64")] +pub enum OutputFrame { + /// Bytes written to stdout + Stdout(String), + /// Bytes written to stderr + Stderr(String), +} + +/// A finished command. +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ExecResponse { + /// Frames in production order + pub output: Vec, + /// Process exit code + pub exit_code: i64, +} + +/// File contents on the way out. +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ReadFileResponse { + /// Contents, base64 because a file is arbitrary bytes + pub contents_base64: String, +} + +#[derive(Clone)] +struct RouteState { + manager: Arc, + sandbox: String, + token: String, + /// Replaced in place on update. Held behind a lock rather than moved into the router so an + /// updated declaration — new limits, image, egress or preview ports — reaches sessions + /// created after it without rebinding the route the workload was already given. + template: Arc>, +} + +/// A running loopback route, and where to reach it. +#[derive(Debug)] +pub struct SandboxRoute { + /// Base URL the binding provider talks to + pub base_url: String, + /// File holding the bearer token. The binding carries this path, never the token itself, + /// so no secret reaches deployment state. + pub token_path: std::path::PathBuf, +} + +/// One serving route, and what is needed to update or stop it. +struct ServingRoute { + base_url: String, + token_path: std::path::PathBuf, + template: Arc>, + /// Dropped to stop the listener. Taken on removal, so a second removal is a no-op. + shutdown: Option>, +} + +/// Routes already serving in this process, keyed by sandbox id. +/// +/// The controller runs its health tick every few seconds, and a route per tick would leak a +/// listener each time and hand the workload a different port than the one it was given. +static ROUTES: OnceLock>> = OnceLock::new(); + +impl SandboxRoute { + /// Returns the sandbox's route, serving it first if this process has not already. + /// + /// Idempotent by sandbox id, so a controller can call it on every step. + pub async fn ensure( + manager: Arc, + sandbox: &str, + template: SandboxSessionConfig, + ) -> Result { + let routes = ROUTES.get_or_init(|| Mutex::new(HashMap::new())); + + { + let serving = routes.lock().expect("no panic holds this lock"); + if let Some(existing) = serving.get(sandbox) { + // An update changes the template, not the address: the workload already holds + // this URL, and rebinding would strand it on a dead port. + *existing.template.lock().expect("no panic holds this lock") = template; + return Ok(Self { + base_url: existing.base_url.clone(), + token_path: existing.token_path.clone(), + }); + } + } + + let (route, template, shutdown) = Self::serve(manager, sandbox, template).await?; + + routes.lock().expect("no panic holds this lock").insert( + sandbox.to_string(), + ServingRoute { + base_url: route.base_url.clone(), + token_path: route.token_path.clone(), + template, + shutdown: Some(shutdown), + }, + ); + + Ok(route) + } + + /// Stops one sandbox's route and removes what it left behind. + /// + /// Delete has to reach further than the containers: a route left serving keeps accepting + /// session creates for a sandbox that no longer exists, and a token file left on disk is a + /// live credential for it. + pub async fn remove(sandbox: &str) { + let Some(routes) = ROUTES.get() else { + return; + }; + + let removed = routes + .lock() + .expect("no panic holds this lock") + .remove(sandbox); + + let Some(mut route) = removed else { + return; + }; + + // Dropping the sender is what the listener's graceful shutdown waits on. + drop(route.shutdown.take()); + + // Best effort: an already-removed token file is the desired end state. + let _ = tokio::fs::remove_file(&route.token_path).await; + } + + /// Binds the route on loopback and serves it until the process ends. + /// + /// Port 0: the OS picks, and the binding learns the address from the resource's outputs. + /// A fixed port would collide between two deployments on one machine. + async fn serve( + manager: Arc, + sandbox: &str, + template: SandboxSessionConfig, + ) -> Result<( + Self, + Arc>, + tokio::sync::oneshot::Sender<()>, + )> { + let token = generate_token(); + let token_path = manager.state_dir().join(token_file_name(sandbox)); + + if let Some(parent) = token_path.parent() { + tokio::fs::create_dir_all(parent) + .await + .into_alien_error() + .map_err(|error| { + AlienError::new(ErrorData::SandboxSessionFailed { + session_id: sandbox.to_string(), + operation: format!("create state directory: {error}"), + }) + })?; + } + + write_token_file(&token_path, &token).await?; + + let template = Arc::new(Mutex::new(template)); + let state = RouteState { + manager, + sandbox: sandbox.to_string(), + token, + template: Arc::clone(&template), + }; + + let router = Router::new() + .route("/v1/sessions", post(create_session).get(list_sessions)) + .route("/v1/sessions/{session_id}", axum::routing::delete(terminate)) + .route("/v1/sessions/{session_id}/exec", post(exec)) + .route("/v1/sessions/{session_id}/files", get(read_file).put(write_file)) + .route("/v1/sessions/{session_id}/preview", get(preview)) + .with_state(state); + + let listener = TcpListener::bind::("127.0.0.1:0".parse().expect("literal")) + .await + .into_alien_error() + .map_err(|error| { + AlienError::new(ErrorData::SandboxSessionFailed { + session_id: sandbox.to_string(), + operation: format!("bind loopback route: {error}"), + }) + })?; + + let address = listener.local_addr().into_alien_error().map_err(|error| { + AlienError::new(ErrorData::SandboxSessionFailed { + session_id: sandbox.to_string(), + operation: format!("read route address: {error}"), + }) + })?; + + let (shutdown, stop) = tokio::sync::oneshot::channel::<()>(); + tokio::spawn(async move { + let serve = axum::serve(listener, router).with_graceful_shutdown(async move { + // Either an explicit stop or the sender being dropped ends the wait. + let _ = stop.await; + }); + if let Err(error) = serve.await { + tracing::error!("local sandbox route stopped: {error}"); + } + }); + + Ok(( + Self { + base_url: format!("http://{address}"), + token_path, + }, + template, + shutdown, + )) + } +} + +/// Writes the token so that it is never readable by another user on the machine, not even for an +/// instant. +/// +/// The mode is set at create time rather than chmod'd after: writing first and restricting second +/// leaves the bearer token world-readable for as long as the two calls take, and this is a +/// credential that grants session creation. +async fn write_token_file(path: &std::path::Path, token: &str) -> Result<()> { + let failed = |error: std::io::Error| { + AlienError::new(ErrorData::SandboxSessionFailed { + session_id: path.display().to_string(), + operation: format!("write token file: {error}"), + }) + }; + + let mut options = tokio::fs::OpenOptions::new(); + options.write(true).create(true).truncate(true); + // tokio's own `mode`, not the std extension trait — the file is created already restricted + // rather than chmod'd after, so the token is never briefly world-readable. + #[cfg(unix)] + options.mode(0o600); + + let mut file = options.open(path).await.map_err(failed)?; + tokio::io::AsyncWriteExt::write_all(&mut file, token.as_bytes()) + .await + .map_err(failed)?; + tokio::io::AsyncWriteExt::flush(&mut file) + .await + .map_err(failed)?; + + Ok(()) +} + +fn generate_token() -> String { + format!("{}{}", uuid::Uuid::new_v4().simple(), uuid::Uuid::new_v4().simple()) +} + +/// Compares in constant time, so a caller cannot recover the token one byte at a time. +fn token_matches(expected: &str, presented: &str) -> bool { + if expected.len() != presented.len() { + return false; + } + + expected + .bytes() + .zip(presented.bytes()) + .fold(0u8, |differences, (a, b)| differences | (a ^ b)) + == 0 +} + +fn authorize(state: &RouteState, headers: &HeaderMap) -> std::result::Result<(), StatusCode> { + let presented = headers + .get(axum::http::header::AUTHORIZATION) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.strip_prefix("Bearer ")) + .ok_or(StatusCode::UNAUTHORIZED)?; + + if token_matches(&state.token, presented) { + Ok(()) + } else { + Err(StatusCode::UNAUTHORIZED) + } +} + +fn failed(error: AlienError) -> (StatusCode, String) { + (StatusCode::BAD_GATEWAY, error.to_string()) +} + +async fn create_session( + State(state): State, + headers: HeaderMap, + Json(body): Json, +) -> std::result::Result, (StatusCode, String)> { + authorize(&state, &headers).map_err(|code| (code, "unauthorized".to_string()))?; + + // Cloned out of the lock: the session create is an await, and the guard is not Send. + let template = state + .template + .lock() + .expect("no panic holds this lock") + .clone(); + + let handle = state + .manager + .create_session(&state.sandbox, &body.session_id, &template) + .await + .map_err(failed)?; + + Ok(Json(SessionBody { + session_id: handle.session_id, + container_id: handle.container_id, + })) +} + +async fn list_sessions( + State(state): State, + headers: HeaderMap, +) -> std::result::Result>, (StatusCode, String)> { + authorize(&state, &headers).map_err(|code| (code, "unauthorized".to_string()))?; + + let sessions = state + .manager + .list_sessions(&state.sandbox) + .await + .map_err(failed)?; + + Ok(Json( + sessions + .into_iter() + .map(|handle| SessionBody { + session_id: handle.session_id, + container_id: handle.container_id, + }) + .collect(), + )) +} + +async fn container_for( + state: &RouteState, + session_id: &str, +) -> std::result::Result { + let sessions = state + .manager + .list_sessions(&state.sandbox) + .await + .map_err(failed)?; + + sessions + .into_iter() + .find(|handle| handle.session_id == session_id) + .map(|handle| handle.container_id) + .ok_or((StatusCode::NOT_FOUND, format!("no session '{session_id}'"))) +} + +async fn exec( + State(state): State, + Path(session_id): Path, + headers: HeaderMap, + Json(body): Json, +) -> std::result::Result, (StatusCode, String)> { + authorize(&state, &headers).map_err(|code| (code, "unauthorized".to_string()))?; + + let container_id = container_for(&state, &session_id).await?; + let result = state + .manager + .exec(&container_id, &body.command) + .await + .map_err(failed)?; + + Ok(Json(ExecResponse { + output: result + .output + .into_iter() + .map(|frame| match frame { + SandboxOutput::Stdout(bytes) => OutputFrame::Stdout(BASE64.encode(bytes)), + SandboxOutput::Stderr(bytes) => OutputFrame::Stderr(BASE64.encode(bytes)), + }) + .collect(), + exit_code: result.exit_code, + })) +} + +async fn write_file( + State(state): State, + Path(session_id): Path, + headers: HeaderMap, + Json(body): Json, +) -> std::result::Result { + authorize(&state, &headers).map_err(|code| (code, "unauthorized".to_string()))?; + + let contents = BASE64 + .decode(body.contents_base64) + .map_err(|error| (StatusCode::BAD_REQUEST, format!("bad base64: {error}")))?; + + let container_id = container_for(&state, &session_id).await?; + state + .manager + .write_file(&container_id, &body.path, &contents) + .await + .map_err(failed)?; + + Ok(StatusCode::NO_CONTENT) +} + +async fn read_file( + State(state): State, + Path(session_id): Path, + Query(query): Query, + headers: HeaderMap, +) -> std::result::Result, (StatusCode, String)> { + authorize(&state, &headers).map_err(|code| (code, "unauthorized".to_string()))?; + + let container_id = container_for(&state, &session_id).await?; + let contents = state + .manager + .read_file(&container_id, &query.path) + .await + .map_err(failed)?; + + Ok(Json(ReadFileResponse { + contents_base64: BASE64.encode(contents), + })) +} + +async fn preview( + State(state): State, + Path(session_id): Path, + Query(query): Query, + headers: HeaderMap, +) -> std::result::Result, (StatusCode, String)> { + authorize(&state, &headers).map_err(|code| (code, "unauthorized".to_string()))?; + + let container_id = container_for(&state, &session_id).await?; + let endpoint = state + .manager + .preview_address(&container_id, query.port) + .await + .map_err(failed)?; + + Ok(Json(PreviewResponse { + endpoint, + allowed_ports: vec![query.port], + })) +} + +async fn terminate( + State(state): State, + Path(session_id): Path, + headers: HeaderMap, +) -> std::result::Result { + authorize(&state, &headers).map_err(|code| (code, "unauthorized".to_string()))?; + + state + .manager + .terminate(&state.sandbox, &session_id) + .await + .map_err(failed)?; + + Ok(StatusCode::NO_CONTENT) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Two sandboxes in one deployment share a state directory. With a fixed file name whichever + /// one started last owned the credential for both, and the other's binding then authenticated + /// with a token that was no longer on disk. + #[test] + fn each_sandbox_owns_its_own_token_file() { + assert_ne!(token_file_name("agents"), token_file_name("runners")); + assert!(token_file_name("agents").contains("agents")); + } + + /// A bearer token that is world-readable even briefly is readable by anything watching the + /// state directory, so the mode has to be set at create time rather than after the write. + #[cfg(unix)] + #[tokio::test] + async fn the_token_file_is_never_readable_by_anyone_else() { + use std::os::unix::fs::PermissionsExt; + + let dir = std::env::temp_dir().join(format!("alien-token-{}", generate_token())); + tokio::fs::create_dir_all(&dir).await.expect("temp dir"); + let path = dir.join("sandbox-manager-agents.token"); + + write_token_file(&path, "secret").await.expect("writes"); + + let mode = tokio::fs::metadata(&path) + .await + .expect("readable") + .permissions() + .mode(); + assert_eq!(mode & 0o777, 0o600, "mode was {:o}", mode & 0o777); + assert_eq!( + tokio::fs::read_to_string(&path).await.expect("contents"), + "secret" + ); + + tokio::fs::remove_dir_all(&dir).await.ok(); + } + + #[test] + fn token_comparison_rejects_wrong_and_short_tokens() { + let token = generate_token(); + + assert!(token_matches(&token, &token)); + assert!(!token_matches(&token, "short")); + assert!(!token_matches(&token, &"0".repeat(token.len()))); + } + + #[test] + fn tokens_are_not_reused_between_routes() { + assert_ne!(generate_token(), generate_token()); + assert!(generate_token().len() >= 64, "a guessable token is not a token"); + } +} diff --git a/crates/alien-local/tests/sandbox_isolation.rs b/crates/alien-local/tests/sandbox_isolation.rs new file mode 100644 index 000000000..eb5145dea --- /dev/null +++ b/crates/alien-local/tests/sandbox_isolation.rs @@ -0,0 +1,466 @@ +//! Real Docker coverage for the local sandbox manager's isolation guarantees. +//! +//! These assert the properties that make the manager safe to point at untrusted code, so they +//! must run against a real daemon — a mock would only re-state the config we passed in. +//! +//! `cargo test -p alien-local --test sandbox_isolation -- --ignored --test-threads=1` + +use std::collections::HashMap; + +use alien_local::{ + LocalSandboxManager, SandboxEgressMode, SandboxOutput, SandboxSessionConfig, +}; +use tempfile::TempDir; + +const IMAGE: &str = "alpine:3.20"; + +fn config(egress: SandboxEgressMode) -> SandboxSessionConfig { + SandboxSessionConfig { + image: IMAGE.to_string(), + cpu_cores: 0.5, + memory_bytes: 256 * 1024 * 1024, + pids_limit: Some(64), + scratch_bytes: 16 * 1024 * 1024, + egress, + preview_ports: Vec::new(), + env: HashMap::new(), + } +} + +fn manager() -> (LocalSandboxManager, TempDir) { + let dir = TempDir::new().expect("temp dir"); + let manager = + LocalSandboxManager::new(dir.path().to_path_buf()).expect("Docker must be reachable"); + (manager, dir) +} + +fn stdout(result: &alien_local::SandboxExecResult) -> String { + result + .output + .iter() + .filter_map(|frame| match frame { + SandboxOutput::Stdout(bytes) => Some(String::from_utf8_lossy(bytes).to_string()), + SandboxOutput::Stderr(_) => None, + }) + .collect() +} + +fn sh(command: &str) -> Vec { + vec!["/bin/sh".to_string(), "-c".to_string(), command.to_string()] +} + +#[tokio::test] +#[ignore = "requires a real Docker daemon"] +async fn session_round_trips_and_runs_unprivileged() { + let (manager, _dir) = manager(); + let sandbox = "isolation-a"; + manager.reap(sandbox).await.expect("clean slate"); + + let session = manager + .create_session(sandbox, "s1", &config(SandboxEgressMode::Deny)) + .await + .expect("session creates"); + + let whoami = manager + .exec(&session.container_id, &sh("id -u")) + .await + .expect("exec runs"); + assert_eq!(whoami.exit_code, 0); + assert_eq!( + stdout(&whoami).trim(), + "65534", + "the workload must not run as root" + ); + + manager + .write_file(&session.container_id, "in.txt", b"payload-in") + .await + .expect("file uploads"); + let read_back = manager + .exec(&session.container_id, &sh("cat /sandbox/in.txt")) + .await + .expect("exec runs"); + assert_eq!(stdout(&read_back).trim(), "payload-in"); + + manager + .exec(&session.container_id, &sh("echo payload-out > /sandbox/out.txt")) + .await + .expect("exec runs"); + let downloaded = manager + .read_file(&session.container_id, "out.txt") + .await + .expect("file downloads"); + assert_eq!(String::from_utf8_lossy(&downloaded).trim(), "payload-out"); + + manager.terminate(sandbox, "s1").await.expect("terminates"); + manager + .terminate(sandbox, "s1") + .await + .expect("terminate is idempotent"); + assert!( + manager.list_sessions(sandbox).await.expect("lists").is_empty(), + "a terminated session must not remain" + ); +} + +#[tokio::test] +#[ignore = "requires a real Docker daemon"] +async fn root_filesystem_is_read_only_and_scratch_is_not() { + let (manager, _dir) = manager(); + let sandbox = "isolation-b"; + manager.reap(sandbox).await.expect("clean slate"); + + let session = manager + .create_session(sandbox, "s1", &config(SandboxEgressMode::Deny)) + .await + .expect("session creates"); + + let root_write = manager + .exec(&session.container_id, &sh("echo x > /escape 2>&1")) + .await + .expect("exec runs"); + assert_ne!( + root_write.exit_code, 0, + "the root filesystem must be read-only, got: {}", + stdout(&root_write) + ); + + let scratch_write = manager + .exec(&session.container_id, &sh("echo x > /sandbox/ok")) + .await + .expect("exec runs"); + assert_eq!(scratch_write.exit_code, 0, "scratch must stay writable"); + + manager.terminate(sandbox, "s1").await.expect("terminates"); +} + +#[tokio::test] +#[ignore = "requires a real Docker daemon"] +async fn deny_egress_has_no_network_at_all() { + let (manager, _dir) = manager(); + let sandbox = "isolation-c"; + manager.reap(sandbox).await.expect("clean slate"); + + let session = manager + .create_session(sandbox, "s1", &config(SandboxEgressMode::Deny)) + .await + .expect("session creates"); + + // Assert on the interface list rather than on a reachability probe: a probe that fails + // could equally mean the network is merely slow. + let interfaces = manager + .exec(&session.container_id, &sh("ls /sys/class/net")) + .await + .expect("exec runs"); + let listed = stdout(&interfaces); + assert!( + !listed.split_whitespace().any(|nic| nic.starts_with("eth")), + "deny must leave no ethernet interface, saw: {listed}" + ); + + manager.terminate(sandbox, "s1").await.expect("terminates"); +} + +#[tokio::test] +#[ignore = "requires a real Docker daemon"] +async fn the_host_gateway_is_not_mapped_in() { + let (manager, _dir) = manager(); + let sandbox = "isolation-d"; + manager.reap(sandbox).await.expect("clean slate"); + + let session = manager + .create_session(sandbox, "s1", &config(SandboxEgressMode::Allow)) + .await + .expect("session creates"); + + // LocalContainerManager maps host.docker.internal:host-gateway into every container. For a + // sandbox that is a route to the developer's machine, so its absence is the assertion. + let hosts = manager + .exec(&session.container_id, &sh("cat /etc/hosts")) + .await + .expect("exec runs"); + assert!( + !stdout(&hosts).contains("host.docker.internal"), + "the host gateway must not be reachable by name: {}", + stdout(&hosts) + ); + + manager.reap(sandbox).await.expect("cleanup"); +} + +#[tokio::test] +#[ignore = "requires a real Docker daemon"] +async fn a_fork_bomb_hits_the_pid_limit_without_taking_the_host_with_it() { + let (manager, _dir) = manager(); + let sandbox = "isolation-e"; + manager.reap(sandbox).await.expect("clean slate"); + + let session = manager + .create_session(sandbox, "s1", &config(SandboxEgressMode::Deny)) + .await + .expect("session creates"); + + // Bounded rather than a true `:(){ :|:& };:` — the point is that the limit binds, and an + // unbounded bomb would leave the assertion at the mercy of the test runner. + let bomb = manager + .exec( + &session.container_id, + &sh("i=0; while [ $i -lt 200 ]; do sleep 30 & i=$((i+1)); done; echo spawned-all"), + ) + .await + .expect("exec runs"); + + assert!( + !stdout(&bomb).contains("spawned-all"), + "the PID limit must bind before 200 processes: {}", + stdout(&bomb) + ); + + // Not asserted: that the session keeps answering. Once the ceiling is reached there is no + // room to fork an exec either, so a follow-up command fails with a runc nsexec error. That + // is the limit working, not the session dying — and the host is unaffected either way, + // which is the property that matters. + + manager.terminate(sandbox, "s1").await.expect("terminates"); +} + +/// Pins what Local can and cannot promise about session-to-session reachability. +/// +/// `deny` is absolute: no interface, so nothing to reach anything with. `allow` is not, and the +/// gap is in the container runtime rather than in this manager — verified with the raw Docker +/// CLI on OrbStack 29.4.0, where two containers on a bridge created with +/// `com.docker.network.bridge.enable_icc=false` still ping each other. Stock Linux Docker +/// honours the option; OrbStack does not, and OrbStack is the common macOS setup. +/// +/// So the promise is: an egress-denied session is isolated, and an egress-allowed session is +/// not isolated from its siblings on every runtime. That is why Local is development-only for +/// untrusted code, and why this is a test rather than a comment. +#[tokio::test] +#[ignore = "requires a real Docker daemon"] +async fn deny_isolates_sessions_and_allow_does_not_promise_to() { + let (manager, _dir) = manager(); + let sandbox = "isolation-f"; + manager.reap(sandbox).await.expect("clean slate"); + + let first = manager + .create_session(sandbox, "s1", &config(SandboxEgressMode::Allow)) + .await + .expect("first session creates"); + let second = manager + .create_session(sandbox, "s2", &config(SandboxEgressMode::Deny)) + .await + .expect("second session creates"); + + let address = manager + .exec(&first.container_id, &sh("hostname -i")) + .await + .expect("exec runs"); + let first_ip = stdout(&address).trim().to_string(); + assert!(!first_ip.is_empty(), "the egress-allowed session needs an address"); + + let from_denied = manager + .exec( + &second.container_id, + &sh(&format!( + "ping -c 1 -W 2 {first_ip} >/dev/null 2>&1 && echo REACHED || echo BLOCKED" + )), + ) + .await + .expect("exec runs"); + assert_eq!( + stdout(&from_denied).trim(), + "BLOCKED", + "an egress-denied session has no interface, so it must reach nothing" + ); + + manager.reap(sandbox).await.expect("cleanup"); +} + +/// Docker accepts port bindings on a network-less container and silently drops them, so this +/// combination would look configured and never resolve. Better a typed error at create. +#[tokio::test] +#[ignore = "requires a real Docker daemon"] +async fn a_preview_port_under_deny_egress_is_refused_rather_than_silently_dropped() { + let (manager, _dir) = manager(); + let sandbox = "isolation-i"; + manager.reap(sandbox).await.expect("clean slate"); + + let mut denied = config(SandboxEgressMode::Deny); + denied.preview_ports = vec![8080]; + + let error = manager + .create_session(sandbox, "s1", &denied) + .await + .expect_err("a preview port with no interface to serve it must be refused"); + assert_eq!(error.code, "SANDBOX_SESSION_FAILED"); + + let mut allowed = config(SandboxEgressMode::Allow); + allowed.preview_ports = vec![8080]; + manager + .create_session(sandbox, "s2", &allowed) + .await + .expect("the same port is fine once there is a network"); + + manager.reap(sandbox).await.expect("cleanup"); +} + +#[tokio::test] +#[ignore = "requires a real Docker daemon"] +async fn reap_removes_every_session_of_a_sandbox_and_leaves_others_alone() { + let (manager, _dir) = manager(); + let mine = "isolation-g"; + let other = "isolation-h"; + manager.reap(mine).await.expect("clean slate"); + manager.reap(other).await.expect("clean slate"); + + manager + .create_session(mine, "s1", &config(SandboxEgressMode::Deny)) + .await + .expect("creates"); + manager + .create_session(mine, "s2", &config(SandboxEgressMode::Deny)) + .await + .expect("creates"); + manager + .create_session(other, "s1", &config(SandboxEgressMode::Deny)) + .await + .expect("creates"); + + assert_eq!(manager.list_sessions(mine).await.expect("lists").len(), 2); + + let reaped = manager.reap(mine).await.expect("reaps"); + assert_eq!(reaped, 2); + assert!(manager.list_sessions(mine).await.expect("lists").is_empty()); + assert_eq!( + manager.list_sessions(other).await.expect("lists").len(), + 1, + "reaping one sandbox must not touch another's sessions" + ); + + manager.reap(other).await.expect("cleanup"); +} + +/// The metadata assertion, with a real bound on what it proves. +/// +/// Under `Deny` this is conclusive: the probe returns "Network unreachable". Under `Allow` on a +/// developer machine there is **no metadata service at that address at all**, so the test cannot +/// distinguish blocked from absent and would pass either way. The endpoint was measured +/// *reachable* from inside a gVisor pod on GKE, so the `Allow` case has to be re-asserted on a +/// cloud host — it is covered here only so the Deny path cannot regress unnoticed. +#[tokio::test] +#[ignore = "requires a real Docker daemon"] +async fn the_metadata_endpoint_is_unreachable_under_both_egress_modes() { + for egress in [SandboxEgressMode::Deny, SandboxEgressMode::Allow] { + let (manager, _dir) = manager(); + let sandbox = format!("meta-{egress:?}").to_lowercase(); + let session = manager + .create_session(&sandbox, "s1", &config(egress)) + .await + .expect("session"); + + let result = manager + .exec( + &session.container_id, + &sh("wget -q -T 3 -O - http://169.254.169.254/ 2>&1; echo rc=$?"), + ) + .await + .expect("exec"); + + let out = stdout(&result); + assert!( + out.contains("rc=1") || out.contains("rc=4") || !out.contains("rc=0"), + "{egress:?}: the metadata endpoint must not answer, got: {out}" + ); + + manager.reap(&sandbox).await.expect("reap"); + } +} + +/// The credential assertion. The manager builds the session's environment from the +/// controller's template, so anything in the host's environment — including whatever cloud +/// credentials the developer happens to be holding — must not appear inside. +#[tokio::test] +#[ignore = "requires a real Docker daemon"] +async fn the_hosts_environment_does_not_leak_into_a_session() { + // Set on the host only. A sandbox that can read it could read a real credential the same way. + std::env::set_var("ALIEN_TEST_HOST_ONLY_SECRET", "host-secret-must-not-appear"); + + let (manager, _dir) = manager(); + let session = manager + .create_session("envleak", "s1", &config(SandboxEgressMode::Deny)) + .await + .expect("session"); + + let result = manager + .exec(&session.container_id, &sh("env")) + .await + .expect("exec"); + + let out = stdout(&result); + assert!( + !out.contains("host-secret-must-not-appear"), + "the host's environment leaked into the sandbox:\n{out}" + ); + assert!( + !out.contains("AWS_SECRET_ACCESS_KEY") && !out.contains("AWS_SESSION_TOKEN"), + "cloud credentials are present in the sandbox environment:\n{out}" + ); + + manager.reap("envleak").await.expect("reap"); +} + +/// No session content reaches anything Alien persists. The session's own output is the caller's; +/// it must not be duplicated into anything Alien keeps. Asserted against the manager's state +/// directory, which is the only thing this layer persists. +#[tokio::test] +#[ignore = "requires a real Docker daemon"] +async fn session_output_is_not_written_into_manager_state() { + let (manager, dir) = manager(); + let session = manager + .create_session("nolog", "s1", &config(SandboxEgressMode::Deny)) + .await + .expect("session"); + + let canary = "canary-9f3a2b7c-session-content"; + let result = manager + .exec(&session.container_id, &sh(&format!("echo {canary}"))) + .await + .expect("exec"); + assert!(stdout(&result).contains(canary), "the caller should get its own output"); + + let mut found = Vec::new(); + for entry in walk(dir.path()) { + if std::fs::read(&entry) + .map(|bytes| String::from_utf8_lossy(&bytes).contains(canary)) + .unwrap_or(false) + { + found.push(entry.display().to_string()); + } + } + + assert!( + found.is_empty(), + "session output was persisted by the manager in: {found:?}" + ); + + manager.reap("nolog").await.expect("reap"); +} + +fn walk(root: &std::path::Path) -> Vec { + let mut out = Vec::new(); + let mut stack = vec![root.to_path_buf()]; + while let Some(path) = stack.pop() { + let Ok(entries) = std::fs::read_dir(&path) else { + continue; + }; + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + stack.push(path); + } else { + out.push(path); + } + } + } + out +} diff --git a/crates/alien-local/tests/sandbox_route.rs b/crates/alien-local/tests/sandbox_route.rs new file mode 100644 index 000000000..53388d431 --- /dev/null +++ b/crates/alien-local/tests/sandbox_route.rs @@ -0,0 +1,315 @@ +//! The local sandbox lifecycle over the real loopback transport. +//! +//! This is the point of building Local first: the same shape the cloud backends carry runs +//! here against a real daemon, over real HTTP, with real auth — no in-process shortcut. +//! +//! `cargo test -p alien-local --test sandbox_route -- --ignored --test-threads=1` + +use std::sync::Arc; + +use alien_local::{LocalSandboxManager, SandboxEgressMode, SandboxRoute, SandboxSessionConfig}; +use base64::engine::general_purpose::STANDARD as BASE64; +use base64::Engine as _; +use serde_json::json; +use tempfile::TempDir; + +const IMAGE: &str = "alpine:3.20"; +const SANDBOX: &str = "route-a"; + +struct Harness { + route: SandboxRoute, + token: String, + client: reqwest::Client, + _dir: TempDir, +} + +impl Harness { + async fn start() -> Self { + let dir = TempDir::new().expect("temp dir"); + let manager = Arc::new( + LocalSandboxManager::new(dir.path().to_path_buf()).expect("Docker must be reachable"), + ); + manager.reap(SANDBOX).await.expect("clean slate"); + + let template = SandboxSessionConfig { + image: IMAGE.to_string(), + cpu_cores: 0.5, + memory_bytes: 268_435_456, + pids_limit: Some(64), + scratch_bytes: 16_777_216, + egress: SandboxEgressMode::Allow, + preview_ports: vec![8080], + env: Default::default(), + }; + + let route = SandboxRoute::ensure(manager, SANDBOX, template) + .await + .expect("route binds on loopback"); + let token = std::fs::read_to_string(&route.token_path).expect("token file is readable"); + + Self { + route, + token, + client: reqwest::Client::new(), + _dir: dir, + } + } + + fn url(&self, path: &str) -> String { + format!("{}{path}", self.route.base_url) + } + + fn authed(&self, request: reqwest::RequestBuilder) -> reqwest::RequestBuilder { + request.bearer_auth(&self.token) + } +} + +#[tokio::test] +#[ignore = "requires a real Docker daemon"] +async fn the_full_session_lifecycle_works_over_the_loopback_route() { + let harness = Harness::start().await; + + let created = harness + .authed(harness.client.post(harness.url("/v1/sessions"))) + .json(&json!({ "sessionId": "s1" })) + .send() + .await + .expect("create request sends"); + assert_eq!(created.status(), 200, "{}", created.text().await.unwrap_or_default()); + + let listed: Vec = harness + .authed(harness.client.get(harness.url("/v1/sessions"))) + .send() + .await + .expect("list sends") + .json() + .await + .expect("list parses"); + assert_eq!(listed.len(), 1, "the created session must be listed"); + + let exec: serde_json::Value = harness + .authed(harness.client.post(harness.url("/v1/sessions/s1/exec"))) + .json(&json!({ "command": ["/bin/sh", "-c", "echo hello-over-http; id -u"] })) + .send() + .await + .expect("exec sends") + .json() + .await + .expect("exec parses"); + + assert_eq!(exec["exitCode"], 0); + let decoded: String = exec["output"] + .as_array() + .expect("frames") + .iter() + .filter(|frame| frame["stream"] == "stdout") + .map(|frame| { + String::from_utf8( + BASE64 + .decode(frame["dataBase64"].as_str().expect("base64 payload")) + .expect("decodes"), + ) + .expect("utf8") + }) + .collect(); + assert!(decoded.contains("hello-over-http"), "got: {decoded}"); + assert!( + decoded.contains("65534"), + "the workload must stay unprivileged across the transport: {decoded}" + ); + + let write = harness + .authed(harness.client.put(harness.url("/v1/sessions/s1/files"))) + .json(&json!({ + "path": "in.txt", + "contentsBase64": BASE64.encode(b"payload-over-http") + })) + .send() + .await + .expect("write sends"); + assert_eq!(write.status(), 204); + + let read: serde_json::Value = harness + .authed( + harness + .client + .get(harness.url("/v1/sessions/s1/files?path=in.txt")), + ) + .send() + .await + .expect("read sends") + .json() + .await + .expect("read parses"); + let contents = BASE64 + .decode(read["contentsBase64"].as_str().expect("base64 payload")) + .expect("decodes"); + assert_eq!(String::from_utf8_lossy(&contents), "payload-over-http"); + + // Preview is a published loopback port, resolved through the authenticated route rather + // than guessed. An undeclared port must not resolve at all. + let raw = harness + .authed(harness.client.get(harness.url("/v1/sessions/s1/preview?port=8080"))) + .send() + .await + .expect("preview sends"); + let status = raw.status(); + let body = raw.text().await.expect("body reads"); + let preview: serde_json::Value = + serde_json::from_str(&body).unwrap_or_else(|_| panic!("preview {status}: {body}")); + let endpoint = preview["endpoint"].as_str().expect("an endpoint"); + assert!( + endpoint.starts_with("http://127.0.0.1:"), + "a preview must be bound to loopback, not the local network: {endpoint}" + ); + + let undeclared = harness + .authed(harness.client.get(harness.url("/v1/sessions/s1/preview?port=9999"))) + .send() + .await + .expect("preview sends"); + assert_ne!( + undeclared.status(), + 200, + "a port not declared at create time must not resolve" + ); + + let terminated = harness + .authed(harness.client.delete(harness.url("/v1/sessions/s1"))) + .send() + .await + .expect("terminate sends"); + assert_eq!(terminated.status(), 204); + + let after: Vec = harness + .authed(harness.client.get(harness.url("/v1/sessions"))) + .send() + .await + .expect("list sends") + .json() + .await + .expect("list parses"); + assert!(after.is_empty(), "a terminated session must not be listed"); +} + +/// The route is on loopback, which is not authorization — anything else on the machine can +/// reach it, and reaching it means running code in someone's sandbox. +#[tokio::test] +#[ignore = "requires a real Docker daemon"] +async fn every_route_refuses_a_request_without_a_valid_token() { + let harness = Harness::start().await; + + let requests = vec![ + harness.client.get(harness.url("/v1/sessions")), + harness + .client + .post(harness.url("/v1/sessions")) + .json(&json!({ "sessionId": "nope" })), + harness + .client + .post(harness.url("/v1/sessions/s1/exec")) + .json(&json!({ "command": ["/bin/sh", "-c", "id"] })), + harness + .client + .get(harness.url("/v1/sessions/s1/files?path=in.txt")), + harness.client.delete(harness.url("/v1/sessions/s1")), + ]; + + for request in requests { + let response = request.send().await.expect("request sends"); + assert_eq!( + response.status(), + 401, + "an unauthenticated request must be refused, got {}", + response.status() + ); + } + + // A wrong token of the right length must fail too, or the check is only testing length. + let wrong = "0".repeat(harness.token.len()); + let response = harness + .client + .get(harness.url("/v1/sessions")) + .bearer_auth(wrong) + .send() + .await + .expect("request sends"); + assert_eq!(response.status(), 401); +} + +/// The template lives server-side, so extra fields in a create request are simply ignored. +/// If they were honoured, an application could raise its own ceilings by asking. +#[tokio::test] +#[ignore = "requires a real Docker daemon"] +async fn an_application_cannot_raise_its_own_limits() { + let harness = Harness::start().await; + + let created = harness + .authed(harness.client.post(harness.url("/v1/sessions"))) + .json(&json!({ + "sessionId": "greedy", + "memoryBytes": 68_719_476_736i64, + "pidsLimit": 1_000_000, + "image": "ubuntu:24.04" + })) + .send() + .await + .expect("create sends"); + assert_eq!(created.status(), 200); + + // The template said alpine with 64 pids; the request asked for ubuntu with a million. + let exec: serde_json::Value = harness + .authed(harness.client.post(harness.url("/v1/sessions/greedy/exec"))) + .json(&json!({ "command": ["/bin/sh", "-c", "cat /etc/os-release | head -1"] })) + .send() + .await + .expect("exec sends") + .json() + .await + .expect("exec parses"); + + let os: String = exec["output"] + .as_array() + .expect("frames") + .iter() + .filter(|frame| frame["stream"] == "stdout") + .map(|frame| { + String::from_utf8( + BASE64 + .decode(frame["dataBase64"].as_str().expect("base64")) + .expect("decodes"), + ) + .expect("utf8") + }) + .collect(); + assert!( + os.to_lowercase().contains("alpine"), + "the template's image must win over the request's: {os}" + ); + + harness + .authed(harness.client.delete(harness.url("/v1/sessions/greedy"))) + .send() + .await + .expect("cleanup"); +} + +#[tokio::test] +#[ignore = "requires a real Docker daemon"] +async fn the_token_file_is_not_world_readable() { + let harness = Harness::start().await; + + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mode = std::fs::metadata(&harness.route.token_path) + .expect("token file exists") + .permissions() + .mode(); + assert_eq!( + mode & 0o077, + 0, + "the token must not be readable by other users, mode was {mode:o}" + ); + } +} diff --git a/crates/alien-manager/src/registry_access.rs b/crates/alien-manager/src/registry_access.rs index 3955db111..7f2a17f50 100644 --- a/crates/alien-manager/src/registry_access.rs +++ b/crates/alien-manager/src/registry_access.rs @@ -808,6 +808,12 @@ mod tests { Err(missing_binding(binding_name)) } + async fn load_sandbox( + &self, + binding_name: &str, + ) -> BindingResult> { + Err(missing_binding(binding_name)) + } } fn aws_state_with_stack(stack: Stack) -> DeploymentState { diff --git a/crates/alien-manager/src/routes/registry_proxy.rs b/crates/alien-manager/src/routes/registry_proxy.rs index 3546e854e..1a36e0057 100644 --- a/crates/alien-manager/src/routes/registry_proxy.rs +++ b/crates/alien-manager/src/routes/registry_proxy.rs @@ -1700,6 +1700,12 @@ mod tests { Err(route_test_error(binding_name)) } + async fn load_sandbox( + &self, + binding_name: &str, + ) -> alien_bindings::error::Result> { + Err(route_test_error(binding_name)) + } } #[test] diff --git a/crates/alien-manager/tests/credentials_mint.rs b/crates/alien-manager/tests/credentials_mint.rs index b8f4c543b..1be0ca917 100644 --- a/crates/alien-manager/tests/credentials_mint.rs +++ b/crates/alien-manager/tests/credentials_mint.rs @@ -183,6 +183,13 @@ impl BindingsProviderApi for FakeServiceAccountProvider { } } + async fn load_sandbox( + &self, + binding_name: &str, + ) -> BindingResult> { + Err(missing_binding(binding_name)) + } + async fn load_storage( &self, binding_name: &str, diff --git a/crates/alien-operator/src/lib.rs b/crates/alien-operator/src/lib.rs index f55032763..94c7a8e55 100644 --- a/crates/alien-operator/src/lib.rs +++ b/crates/alien-operator/src/lib.rs @@ -149,6 +149,7 @@ pub async fn run_operator_with_cancel_and_loops( let otlp_db = db.clone(); let otlp_namespace = config.namespace.clone(); let otlp_collector_token = config.collector_token.clone(); + let sandbox_broker = sandbox_broker_router(&config).await; let otlp_cancel = cancel.clone(); tokio::spawn(async move { if let Err(e) = otlp_server::start_otlp_server( @@ -157,6 +158,7 @@ pub async fn run_operator_with_cancel_and_loops( otlp_db, otlp_namespace, otlp_collector_token, + sandbox_broker, otlp_cancel, ) .await @@ -542,3 +544,25 @@ mod tests { unsafe { libc::kill(pid as libc::pid_t, 0) == 0 } } } + +/// Builds the sandbox broker's routes, when this operator runs a Kubernetes deployment. +/// +/// `None` everywhere else: the broker claims pods, so it exists only where there are pods to +/// claim. A state it cannot build disables the broker rather than taking the operator down — a +/// deployment with no sandbox is unaffected, and one with a sandbox reports it on first claim +/// rather than by refusing to boot. +async fn sandbox_broker_router(config: &config::OperatorConfig) -> Option { + if config.platform != alien_core::Platform::Kubernetes { + return None; + } + + let namespace = config.namespace.clone()?; + + match alien_infra::BrokerState::in_cluster(namespace).await { + Ok(state) => Some(alien_infra::broker_router(state)), + Err(error) => { + tracing::warn!(error = %error, "sandbox broker disabled: no in-cluster Kubernetes client"); + None + } + } +} diff --git a/crates/alien-operator/src/otlp_server.rs b/crates/alien-operator/src/otlp_server.rs index 80f4cea74..b44969a33 100644 --- a/crates/alien-operator/src/otlp_server.rs +++ b/crates/alien-operator/src/otlp_server.rs @@ -46,6 +46,7 @@ pub async fn start_otlp_server( db: Arc, namespace: Option, collector_token: Option, + sandbox_broker: Option, cancel: CancellationToken, ) -> crate::error::Result<()> { let addr = SocketAddr::new(host, port); @@ -65,6 +66,14 @@ pub async fn start_otlp_server( collector_token, }); + // The sandbox broker shares this server because the chart already exposes this port through + // the operator's Service. A second listener would need a second port and a chart change to + // reach the same pods. + let app = match sandbox_broker { + Some(broker) => app.merge(broker), + None => app, + }; + let listener = tokio::net::TcpListener::bind(&addr) .await .into_alien_error() @@ -204,6 +213,7 @@ mod tests { db, None, None, + None, server_cancel, ) .await diff --git a/packages/core/src/generated/index.ts b/packages/core/src/generated/index.ts index f7fad2457..2033f7b70 100644 --- a/packages/core/src/generated/index.ts +++ b/packages/core/src/generated/index.ts @@ -48,6 +48,7 @@ export type { AwsRemoteBindingsImportData } from "./zod/aws-remote-bindings-impo export type { AwsRemoteStackManagementHeartbeatData } from "./zod/aws-remote-stack-management-heartbeat-data-schema.js"; export type { AwsRemoteStackManagementImportData } from "./zod/aws-remote-stack-management-import-data-schema.js"; export type { AwsS3StorageHeartbeatData } from "./zod/aws-s3-storage-heartbeat-data-schema.js"; +export type { AwsSandboxImportData } from "./zod/aws-sandbox-import-data-schema.js"; export type { AwsServiceAccountImportData } from "./zod/aws-service-account-import-data-schema.js"; export type { AwsSqsQueueHeartbeatData } from "./zod/aws-sqs-queue-heartbeat-data-schema.js"; export type { AwsStorageImportData } from "./zod/aws-storage-import-data-schema.js"; @@ -86,6 +87,7 @@ export type { AzureResourceGroupHeartbeatStatus } from "./zod/azure-resource-gro export type { AzureResourceGroupImportData } from "./zod/azure-resource-group-import-data-schema.js"; export type { AzureResourceProviderActivationHeartbeatData } from "./zod/azure-resource-provider-activation-heartbeat-data-schema.js"; export type { AzureSandboxGroupHeartbeatData } from "./zod/azure-sandbox-group-heartbeat-data-schema.js"; +export type { AzureSandboxImportData } from "./zod/azure-sandbox-import-data-schema.js"; export type { AzureServiceAccountImportData } from "./zod/azure-service-account-import-data-schema.js"; export type { AzureServiceActivationImportData } from "./zod/azure-service-activation-import-data-schema.js"; export type { AzureServiceBusNamespaceHeartbeatData } from "./zod/azure-service-bus-namespace-heartbeat-data-schema.js"; @@ -450,6 +452,7 @@ export { AwsRemoteBindingsImportDataSchema } from "./zod/aws-remote-bindings-imp export { AwsRemoteStackManagementHeartbeatDataSchema } from "./zod/aws-remote-stack-management-heartbeat-data-schema.js"; export { AwsRemoteStackManagementImportDataSchema } from "./zod/aws-remote-stack-management-import-data-schema.js"; export { AwsS3StorageHeartbeatDataSchema } from "./zod/aws-s3-storage-heartbeat-data-schema.js"; +export { AwsSandboxImportDataSchema } from "./zod/aws-sandbox-import-data-schema.js"; export { AwsServiceAccountImportDataSchema } from "./zod/aws-service-account-import-data-schema.js"; export { AwsSqsQueueHeartbeatDataSchema } from "./zod/aws-sqs-queue-heartbeat-data-schema.js"; export { AwsStorageImportDataSchema } from "./zod/aws-storage-import-data-schema.js"; @@ -488,6 +491,7 @@ export { AzureResourceGroupHeartbeatStatusSchema } from "./zod/azure-resource-gr export { AzureResourceGroupImportDataSchema } from "./zod/azure-resource-group-import-data-schema.js"; export { AzureResourceProviderActivationHeartbeatDataSchema } from "./zod/azure-resource-provider-activation-heartbeat-data-schema.js"; export { AzureSandboxGroupHeartbeatDataSchema } from "./zod/azure-sandbox-group-heartbeat-data-schema.js"; +export { AzureSandboxImportDataSchema } from "./zod/azure-sandbox-import-data-schema.js"; export { AzureServiceAccountImportDataSchema } from "./zod/azure-service-account-import-data-schema.js"; export { AzureServiceActivationImportDataSchema } from "./zod/azure-service-activation-import-data-schema.js"; export { AzureServiceBusNamespaceHeartbeatDataSchema } from "./zod/azure-service-bus-namespace-heartbeat-data-schema.js"; diff --git a/packages/core/src/generated/schemas/awsSandboxImportData.json b/packages/core/src/generated/schemas/awsSandboxImportData.json new file mode 100644 index 000000000..ac26120c9 --- /dev/null +++ b/packages/core/src/generated/schemas/awsSandboxImportData.json @@ -0,0 +1 @@ +{"type":"object","description":"AWS Sandbox ImportData.\n\nCarries the Frozen parent from the setup emitter to the runtime controller. The image\n**version** is not decoration: `RunMicrovm` has no `tags`, so image plus version is the only\nsession identity there is, and a controller holding a stale version would enumerate the wrong\nset and orphan every session started on the previous one.","required":["imageIdentifier","imageArn","imageVersion"],"properties":{"egressConnectorArns":{"type":"array","items":{"type":"string"},"description":"Egress network connectors. Deleting one while MicroVMs still reference it breaks their\nnetworking, so teardown needs them named rather than rediscovered."},"executionRoleArn":{"type":["string","null"],"description":"Execution role attached to each MicroVM, distinct from the workload's own role."},"imageArn":{"type":"string","description":"MicroVM image ARN."},"imageIdentifier":{"type":"string","description":"MicroVM image identifier."},"imageVersion":{"type":"string","description":"Image version the sessions are scoped to. Re-imported on every image roll."},"previewPorts":{"type":"array","items":{"type":"integer","format":"int32","minimum":0},"description":"Ports a preview capability may be minted for; empty means preview is not offered."}},"x-readme-ref-name":"AwsSandboxImportData"} \ No newline at end of file diff --git a/packages/core/src/generated/schemas/azureSandboxImportData.json b/packages/core/src/generated/schemas/azureSandboxImportData.json new file mode 100644 index 000000000..3ceafa8a8 --- /dev/null +++ b/packages/core/src/generated/schemas/azureSandboxImportData.json @@ -0,0 +1 @@ +{"type":"object","description":"Azure Sandbox ImportData.\n\nCarries the sandbox group from the setup emitter to the runtime controller. All three fields\nare required to address it: the ADC data plane endpoint is **per-region**, so a group without\nits region cannot be reached at all, and the data plane path is scoped by resource group.","required":["sandboxGroup","region","resourceGroup"],"properties":{"region":{"type":"string","description":"Region the group lives in; selects the ADC endpoint."},"resourceGroup":{"type":"string","description":"Resource group containing the sandbox group."},"sandboxGroup":{"type":"string","description":"Sandbox group name."}},"x-readme-ref-name":"AzureSandboxImportData"} \ No newline at end of file diff --git a/packages/core/src/generated/zod/aws-sandbox-import-data-schema.ts b/packages/core/src/generated/zod/aws-sandbox-import-data-schema.ts new file mode 100644 index 000000000..0078f7b13 --- /dev/null +++ b/packages/core/src/generated/zod/aws-sandbox-import-data-schema.ts @@ -0,0 +1,20 @@ +/** +* Generated by Kubb (https://kubb.dev/). +* Do not edit manually. +*/ + +import * as z from "zod"; + +/** + * @description AWS Sandbox ImportData.\n\nCarries the Frozen parent from the setup emitter to the runtime controller. The image\n**version** is not decoration: `RunMicrovm` has no `tags`, so image plus version is the only\nsession identity there is, and a controller holding a stale version would enumerate the wrong\nset and orphan every session started on the previous one. + */ +export const AwsSandboxImportDataSchema = z.object({ + "egressConnectorArns": z.optional(z.array(z.string()).describe("Egress network connectors. Deleting one while MicroVMs still reference it breaks their\nnetworking, so teardown needs them named rather than rediscovered.")), +"executionRoleArn": z.string().describe("Execution role attached to each MicroVM, distinct from the workload's own role.").nullish(), +"imageArn": z.string().describe("MicroVM image ARN."), +"imageIdentifier": z.string().describe("MicroVM image identifier."), +"imageVersion": z.string().describe("Image version the sessions are scoped to. Re-imported on every image roll."), +"previewPorts": z.optional(z.array(z.int().min(0)).describe("Ports a preview capability may be minted for; empty means preview is not offered.")) + }).describe("AWS Sandbox ImportData.\n\nCarries the Frozen parent from the setup emitter to the runtime controller. The image\n**version** is not decoration: `RunMicrovm` has no `tags`, so image plus version is the only\nsession identity there is, and a controller holding a stale version would enumerate the wrong\nset and orphan every session started on the previous one.") + +export type AwsSandboxImportData = z.infer \ No newline at end of file diff --git a/packages/core/src/generated/zod/azure-sandbox-import-data-schema.ts b/packages/core/src/generated/zod/azure-sandbox-import-data-schema.ts new file mode 100644 index 000000000..9fd6aaa41 --- /dev/null +++ b/packages/core/src/generated/zod/azure-sandbox-import-data-schema.ts @@ -0,0 +1,17 @@ +/** +* Generated by Kubb (https://kubb.dev/). +* Do not edit manually. +*/ + +import * as z from "zod"; + +/** + * @description Azure Sandbox ImportData.\n\nCarries the sandbox group from the setup emitter to the runtime controller. All three fields\nare required to address it: the ADC data plane endpoint is **per-region**, so a group without\nits region cannot be reached at all, and the data plane path is scoped by resource group. + */ +export const AzureSandboxImportDataSchema = z.object({ + "region": z.string().describe("Region the group lives in; selects the ADC endpoint."), +"resourceGroup": z.string().describe("Resource group containing the sandbox group."), +"sandboxGroup": z.string().describe("Sandbox group name.") + }).describe("Azure Sandbox ImportData.\n\nCarries the sandbox group from the setup emitter to the runtime controller. All three fields\nare required to address it: the ADC data plane endpoint is **per-region**, so a group without\nits region cannot be reached at all, and the data plane path is scoped by resource group.") + +export type AzureSandboxImportData = z.infer \ No newline at end of file diff --git a/packages/core/src/generated/zod/index.ts b/packages/core/src/generated/zod/index.ts index b68351442..e16489ab8 100644 --- a/packages/core/src/generated/zod/index.ts +++ b/packages/core/src/generated/zod/index.ts @@ -48,6 +48,7 @@ export type { AwsRemoteBindingsImportData } from "./aws-remote-bindings-import-d export type { AwsRemoteStackManagementHeartbeatData } from "./aws-remote-stack-management-heartbeat-data-schema.js"; export type { AwsRemoteStackManagementImportData } from "./aws-remote-stack-management-import-data-schema.js"; export type { AwsS3StorageHeartbeatData } from "./aws-s3-storage-heartbeat-data-schema.js"; +export type { AwsSandboxImportData } from "./aws-sandbox-import-data-schema.js"; export type { AwsServiceAccountImportData } from "./aws-service-account-import-data-schema.js"; export type { AwsSqsQueueHeartbeatData } from "./aws-sqs-queue-heartbeat-data-schema.js"; export type { AwsStorageImportData } from "./aws-storage-import-data-schema.js"; @@ -86,6 +87,7 @@ export type { AzureResourceGroupHeartbeatStatus } from "./azure-resource-group-h export type { AzureResourceGroupImportData } from "./azure-resource-group-import-data-schema.js"; export type { AzureResourceProviderActivationHeartbeatData } from "./azure-resource-provider-activation-heartbeat-data-schema.js"; export type { AzureSandboxGroupHeartbeatData } from "./azure-sandbox-group-heartbeat-data-schema.js"; +export type { AzureSandboxImportData } from "./azure-sandbox-import-data-schema.js"; export type { AzureServiceAccountImportData } from "./azure-service-account-import-data-schema.js"; export type { AzureServiceActivationImportData } from "./azure-service-activation-import-data-schema.js"; export type { AzureServiceBusNamespaceHeartbeatData } from "./azure-service-bus-namespace-heartbeat-data-schema.js"; @@ -450,6 +452,7 @@ export { AwsRemoteBindingsImportDataSchema } from "./aws-remote-bindings-import- export { AwsRemoteStackManagementHeartbeatDataSchema } from "./aws-remote-stack-management-heartbeat-data-schema.js"; export { AwsRemoteStackManagementImportDataSchema } from "./aws-remote-stack-management-import-data-schema.js"; export { AwsS3StorageHeartbeatDataSchema } from "./aws-s3-storage-heartbeat-data-schema.js"; +export { AwsSandboxImportDataSchema } from "./aws-sandbox-import-data-schema.js"; export { AwsServiceAccountImportDataSchema } from "./aws-service-account-import-data-schema.js"; export { AwsSqsQueueHeartbeatDataSchema } from "./aws-sqs-queue-heartbeat-data-schema.js"; export { AwsStorageImportDataSchema } from "./aws-storage-import-data-schema.js"; @@ -488,6 +491,7 @@ export { AzureResourceGroupHeartbeatStatusSchema } from "./azure-resource-group- export { AzureResourceGroupImportDataSchema } from "./azure-resource-group-import-data-schema.js"; export { AzureResourceProviderActivationHeartbeatDataSchema } from "./azure-resource-provider-activation-heartbeat-data-schema.js"; export { AzureSandboxGroupHeartbeatDataSchema } from "./azure-sandbox-group-heartbeat-data-schema.js"; +export { AzureSandboxImportDataSchema } from "./azure-sandbox-import-data-schema.js"; export { AzureServiceAccountImportDataSchema } from "./azure-service-account-import-data-schema.js"; export { AzureServiceActivationImportDataSchema } from "./azure-service-activation-import-data-schema.js"; export { AzureServiceBusNamespaceHeartbeatDataSchema } from "./azure-service-bus-namespace-heartbeat-data-schema.js";