diff --git a/docs/bridge-networking.md b/docs/bridge-networking.md index 3e852e239..2f70cd45a 100644 --- a/docs/bridge-networking.md +++ b/docs/bridge-networking.md @@ -19,8 +19,23 @@ allowed_modes = ["user", "bridge"] bridge = "virbr0" # Optional additional bridges that VM requests may select. allowed_bridges = ["dstack-br1"] +anti_spoof = true +netd_socket = "/run/dstack/netd.sock" +isolate_bridge_ports = true ``` +With `anti_spoof = true`, run the privileged backend before starting the VMM: + +```bash +sudo dstack-vmm --config /etc/dstack/vmm.toml netd +``` + +The VMM fails closed if it cannot prepare a protected TAP. The backend creates +a deterministic TAP, locks bridge learning to the assigned MAC, installs an +nftables netdev-ingress policy, and returns the TAP name to QEMU. The default +socket is root-only. For an unprivileged VMM, configure `netd_socket_gid` and +`netd_allowed_uids`, and run both processes under matching service identities. + ### Per-VM override Individual VMs can override the global networking mode via: diff --git a/dstack/vmm/Cargo.toml b/dstack/vmm/Cargo.toml index 92a9a8b12..ea45241ef 100644 --- a/dstack/vmm/Cargo.toml +++ b/dstack/vmm/Cargo.toml @@ -25,7 +25,7 @@ sha2.workspace = true hex.workspace = true fs-err.workspace = true getrandom = { workspace = true, features = ["std"] } -nix = { workspace = true, features = ["user"] } +nix = { workspace = true, features = ["fs", "user"] } dirs.workspace = true which.workspace = true clap = { workspace = true, features = ["derive", "string"] } diff --git a/dstack/vmm/src/app.rs b/dstack/vmm/src/app.rs index 2939cee27..593012040 100644 --- a/dstack/vmm/src/app.rs +++ b/dstack/vmm/src/app.rs @@ -30,6 +30,8 @@ use std::time::SystemTime; use supervisor_client::SupervisorClient; use tracing::{debug, error, info, warn}; +use crate::netd::{Client as NetdClient, NicAttachment}; + pub use image::{Image, ImageInfo}; pub(crate) use network::{ resolve_networking, resolved_networks, validate_resolved_network, validate_resolved_networks, @@ -71,6 +73,23 @@ fn signal_pidfd(pid: u32, signal: libc::c_int) -> std::io::Result<()> { } } +fn protected_attachment( + vm_id: &str, + networking: &Networking, + nic_index: usize, +) -> Option { + if !networking.is_bridge() || !networking.anti_spoof || networking.tap.is_empty() { + return None; + } + Some(NicAttachment { + vm_id: vm_id.to_string(), + nic_index, + bridge: networking.bridge.clone(), + tap: networking.tap.clone(), + mac: network::mac_address_for_vm_index(vm_id, &networking.mac_prefix_bytes(), nic_index), + }) +} + #[derive(Deserialize, Serialize, Debug, Clone)] pub struct PortMapping { pub address: IpAddr, @@ -305,6 +324,71 @@ impl App { VmWorkDir::new(self.config.run_path.join(id)) } + async fn prepare_protected_networks( + &self, + vm_id: &str, + networks: &mut [Networking], + ) -> Result<()> { + let client = NetdClient::new(&self.config.cvm.networking.netd_socket); + let mut prepared = Vec::new(); + for (index, networking) in networks.iter_mut().enumerate() { + if !networking.is_bridge() || !networking.anti_spoof { + continue; + } + let mac = + network::mac_address_for_vm_index(vm_id, &networking.mac_prefix_bytes(), index); + match client.prepare(vm_id, index, &networking.bridge, &mac).await { + Ok(attachment) => { + networking.tap = attachment.tap.clone(); + prepared.push(attachment); + } + Err(error) => { + for attachment in prepared.iter().rev() { + if let Err(cleanup_error) = client.remove(attachment).await { + warn!( + vm_id, + tap = %attachment.tap, + "failed to roll back protected NIC: {cleanup_error:#}" + ); + } + } + return Err(error).context("failed to prepare protected bridge networking"); + } + } + } + Ok(()) + } + + async fn remove_protected_networks(&self, vm_id: &str, networks: &[Networking]) { + let client = NetdClient::new(&self.config.cvm.networking.netd_socket); + for (index, networking) in networks.iter().enumerate().rev() { + let Some(attachment) = protected_attachment(vm_id, networking, index) else { + continue; + }; + if let Err(error) = client.remove(&attachment).await { + warn!( + vm_id = %attachment.vm_id, + tap = %attachment.tap, + "failed to remove protected NIC: {error:#}" + ); + } + } + } + + async fn check_protected_networks(&self, vm_id: &str, networks: &[Networking]) -> Result<()> { + let client = NetdClient::new(&self.config.cvm.networking.netd_socket); + for (index, networking) in networks.iter().enumerate() { + let Some(attachment) = protected_attachment(vm_id, networking, index) else { + continue; + }; + client + .check(&attachment) + .await + .with_context(|| format!("protected NIC {} is not enforced", attachment.tap))?; + } + Ok(()) + } + pub fn new(config: Config, supervisor: SupervisorClient) -> Self { let cid_start = config.cvm.cid_start; let cid_end = cid_start.saturating_add(config.cvm.cid_pool_size); @@ -424,16 +508,34 @@ impl App { append_boot_separator(&work_dir.stderr_file()); let devices = self.try_allocate_gpus(&vm_config.manifest)?; - let processes = vm_config.config_qemu(&work_dir, &self.config.cvm, &devices)?; - let runtime_networks = resolved_networks(&vm_config.manifest, &self.config.cvm); - work_dir.set_runtime_networks(&runtime_networks)?; + let mut runtime_networks = resolved_networks(&vm_config.manifest, &self.config.cvm); + self.prepare_protected_networks(id, &mut runtime_networks) + .await?; + let processes = match vm_config.config_qemu_with_networks( + &work_dir, + &self.config.cvm, + &devices, + Some(&runtime_networks), + ) { + Ok(processes) => processes, + Err(error) => { + self.remove_protected_networks(id, &runtime_networks).await; + return Err(error); + } + }; + if let Err(error) = work_dir.set_runtime_networks(&runtime_networks) { + self.remove_protected_networks(id, &runtime_networks).await; + return Err(error); + } { let mut state = self.lock(); let vm_state = state.get_mut(id).context("VM not found")?; - vm_state.state.runtime_networks = runtime_networks; + vm_state.state.runtime_networks = runtime_networks.clone(); } for process in processes { if let Err(err) = self.supervisor.deploy(&process).await { + let _ = self.supervisor.stop(id).await; + self.remove_protected_networks(id, &runtime_networks).await; if let Err(clear_err) = work_dir.clear_runtime_networks() { warn!( id, @@ -465,6 +567,8 @@ impl App { pub async fn stop_vm(&self, id: &str) -> Result<()> { self.set_started(id, false)?; self.stop_vm_process(id).await?; + let networks = self.work_dir(id).runtime_networks(); + self.remove_protected_networks(id, &networks).await; Ok(()) } @@ -574,6 +678,9 @@ impl App { } } + let runtime_networks = self.work_dir(id).runtime_networks(); + self.remove_protected_networks(id, &runtime_networks).await; + // Only delete the workdir for user-initiated removal or if .removing marker exists. // Orphaned supervisor processes without the marker keep their data intact. let vm_path = self.work_dir(id); @@ -672,6 +779,22 @@ impl App { } } + // A protected guest must never continue running when its host-side + // enforcement cannot be proven after VMM restart. + for id in occupied_cids.keys() { + if !self.lock().vms.contains_key(id) { + continue; + } + let networks = self.work_dir(id).runtime_networks(); + if let Err(error) = self.check_protected_networks(id, &networks).await { + error!(id, "protected networking reconciliation failed: {error:#}"); + self.stop_vm_process(id) + .await + .with_context(|| format!("failed to quiesce VM {id}"))?; + self.remove_protected_networks(id, &networks).await; + } + } + // Resume cleanup for VMs with .removing marker for id in removing_ids { self.spawn_finish_remove(&id); @@ -1610,6 +1733,12 @@ mod tests { allowed_modes: vec![], bridge: "dstack-br0".to_string(), allowed_bridges: vec![], + anti_spoof: false, + netd_socket: Default::default(), + netd_allowed_uids: vec![], + netd_socket_gid: None, + isolate_bridge_ports: true, + tap: String::new(), mac_prefix: String::new(), net: String::new(), dhcp_start: String::new(), @@ -1829,6 +1958,12 @@ mod tests { allowed_modes: vec![], bridge: "dstack-br0".to_string(), allowed_bridges: vec![], + anti_spoof: false, + netd_socket: Default::default(), + netd_allowed_uids: vec![], + netd_socket_gid: None, + isolate_bridge_ports: true, + tap: String::new(), mac_prefix: "02:aa:bb".to_string(), net: String::new(), dhcp_start: String::new(), diff --git a/dstack/vmm/src/app/network.rs b/dstack/vmm/src/app/network.rs index 980ea1410..34e9a6cd4 100644 --- a/dstack/vmm/src/app/network.rs +++ b/dstack/vmm/src/app/network.rs @@ -159,6 +159,12 @@ mod tests { allowed_modes: vec![], bridge: bridge.to_string(), allowed_bridges: vec![], + anti_spoof: false, + netd_socket: Default::default(), + netd_allowed_uids: vec![], + netd_socket_gid: None, + isolate_bridge_ports: true, + tap: String::new(), mac_prefix: String::new(), net: String::new(), dhcp_start: String::new(), diff --git a/dstack/vmm/src/app/qemu.rs b/dstack/vmm/src/app/qemu.rs index 754a1ef90..8c41d8b14 100644 --- a/dstack/vmm/src/app/qemu.rs +++ b/dstack/vmm/src/app/qemu.rs @@ -198,13 +198,16 @@ impl PreparedQemuLaunch { workdir: impl AsRef, cfg: &CvmConfig, gpus: &GpuConfig, + prepared_networks: Option<&[Networking]>, ) -> Result { let workdir = VmWorkDir::new(workdir); prepare_data_disk(vm, &workdir, cfg)?; prepare_shared_dir(&workdir)?; let app_compose = workdir.app_compose().context("failed to get app compose")?; let platform = cfg.resolved_platform(); - let networks = resolved_networks(&vm.manifest, cfg); + let networks = prepared_networks + .map(<[Networking]>::to_vec) + .unwrap_or_else(|| resolved_networks(&vm.manifest, cfg)); validate_resolved_networks(&networks, &cfg.networking)?; let volumes = vm .manifest @@ -340,7 +343,17 @@ impl VmConfig { cfg: &CvmConfig, gpus: &GpuConfig, ) -> Result> { - let prepared = PreparedQemuLaunch::prepare(self, workdir, cfg, gpus)?; + self.config_qemu_with_networks(workdir, cfg, gpus, None) + } + + pub fn config_qemu_with_networks( + &self, + workdir: impl AsRef, + cfg: &CvmConfig, + gpus: &GpuConfig, + networks: Option<&[Networking]>, + ) -> Result> { + let prepared = PreparedQemuLaunch::prepare(self, workdir, cfg, gpus, networks)?; let process = QemuCommandBuilder { vm: self, cfg, @@ -600,7 +613,17 @@ impl QemuCommandBuilder<'_> { } NetworkingMode::Bridge => { tracing::info!("bridge networking: mac={mac} bridge={}", networking.bridge); - format!("bridge,id={net_id},br={}", networking.bridge) + if networking.anti_spoof { + if networking.tap.is_empty() { + bail!("protected bridge networking is missing a prepared TAP"); + } + format!( + "tap,id={net_id},ifname={},script=no,downscript=no,vhost=off", + networking.tap + ) + } else { + format!("bridge,id={net_id},br={}", networking.bridge) + } } NetworkingMode::Custom => { if !networking.netdev.contains(&format!("id={net_id}")) { diff --git a/dstack/vmm/src/config.rs b/dstack/vmm/src/config.rs index 98264e32f..815509bbd 100644 --- a/dstack/vmm/src/config.rs +++ b/dstack/vmm/src/config.rs @@ -523,6 +523,14 @@ pub enum NetworkingMode { Custom, } +fn default_netd_socket() -> PathBuf { + PathBuf::from("/run/dstack/netd.sock") +} + +fn default_true() -> bool { + true +} + /// Flat networking configuration. The `mode` field selects which backend is /// active; the remaining fields are only relevant for their respective mode /// and carry serde defaults so they can be omitted in the config file. @@ -545,6 +553,31 @@ pub struct Networking { #[serde(default, skip_serializing)] pub allowed_bridges: Vec, + /// Enable host-enforced MAC and L2 filtering through `dstack-netd`. + #[serde(default)] + pub anti_spoof: bool, + + /// Unix socket exposed by the privileged networking backend. + #[serde(default = "default_netd_socket", skip_serializing)] + pub netd_socket: PathBuf, + + /// UIDs authorized to call `dstack-netd`. Empty authorizes only root. + #[serde(default, skip_serializing)] + pub netd_allowed_uids: Vec, + + /// Optional group owner for the netd socket. When set, the socket mode is + /// `0660`; otherwise it remains root-only `0600`. + #[serde(default, skip_serializing)] + pub netd_socket_gid: Option, + + /// Prevent forwarding between protected bridge ports. + #[serde(default = "default_true")] + pub isolate_bridge_ports: bool, + + /// Runtime-only deterministic TAP name returned by `dstack-netd`. + #[serde(default, skip_serializing_if = "String::is_empty")] + pub tap: String, + // ── MAC prefix ───────────────────────────────────────────────── /// Fixed MAC address prefix (0-3 colon-separated hex bytes, e.g. "02:ab:cd"). /// Remaining bytes are derived from the VM ID hash. diff --git a/dstack/vmm/src/main.rs b/dstack/vmm/src/main.rs index fb34bc362..b35cc338a 100644 --- a/dstack/vmm/src/main.rs +++ b/dstack/vmm/src/main.rs @@ -28,6 +28,7 @@ mod guest_api_service; mod host_api_service; mod main_routes; mod main_service; +mod netd; mod one_shot; mod openapi; mod vm_launcher; @@ -60,6 +61,8 @@ enum Command { /// Internal per-VM QEMU/swtpm launcher. #[command(hide = true)] VmLauncher(VmLauncherArgs), + /// Run the privileged bridge networking backend. + Netd, } #[derive(ClapArgs)] @@ -188,6 +191,7 @@ async fn main() -> Result<()> { // Handle commands match args.command.unwrap_or_default() { Command::VmLauncher(_) => unreachable!("launcher mode handled before config loading"), + Command::Netd => return netd::serve(config.cvm).await, Command::Run(run_args) => { // One-shot VM execution mode return one_shot::run_one_shot( diff --git a/dstack/vmm/src/main_service.rs b/dstack/vmm/src/main_service.rs index e8213b303..b273e9a53 100644 --- a/dstack/vmm/src/main_service.rs +++ b/dstack/vmm/src/main_service.rs @@ -341,6 +341,12 @@ fn networking_from_proto(proto: &rpc::NetworkingConfig) -> Result +// +// SPDX-License-Identifier: Apache-2.0 + +//! Least-privilege host networking backend for protected bridge NICs. + +use std::{ + fs, + io::Write, + os::unix::{fs::PermissionsExt, net::UnixStream as StdUnixStream}, + path::{Path, PathBuf}, + process::{Command, Stdio}, +}; + +use anyhow::{bail, Context, Result}; +use nix::unistd::{chown, Gid, User}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use tokio::{ + io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader as AsyncBufReader}, + net::{UnixListener, UnixStream}, +}; + +use crate::{ + app::validate_resolved_network, + config::{CvmConfig, NetworkingMode}, +}; + +const MAX_REQUEST_BYTES: usize = 4096; +const NFT_TABLE_PREFIX: &str = "dstack_"; +const TAP_ALIAS_PREFIX: &str = "dstack-vmm:"; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub(crate) struct NicAttachment { + pub vm_id: String, + pub nic_index: usize, + pub bridge: String, + pub tap: String, + pub mac: String, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(tag = "action", rename_all = "snake_case")] +enum Request { + Prepare { + vm_id: String, + nic_index: usize, + bridge: String, + mac: String, + }, + Remove { + attachment: NicAttachment, + }, + Check { + attachment: NicAttachment, + }, +} + +#[derive(Debug, Serialize, Deserialize)] +struct Response { + ok: bool, + attachment: Option, + error: Option, +} + +impl Response { + fn success(attachment: Option) -> Self { + Self { + ok: true, + attachment, + error: None, + } + } + + fn error(error: impl ToString) -> Self { + Self { + ok: false, + attachment: None, + error: Some(error.to_string()), + } + } + + fn into_result(self) -> Result> { + if self.ok { + Ok(self.attachment) + } else { + bail!(self.error.unwrap_or_else(|| "dstack-netd failed".into())) + } + } +} + +#[derive(Clone)] +pub(crate) struct Client { + socket: PathBuf, +} + +impl Client { + pub(crate) fn new(socket: impl Into) -> Self { + Self { + socket: socket.into(), + } + } + + pub(crate) async fn prepare( + &self, + vm_id: &str, + nic_index: usize, + bridge: &str, + mac: &str, + ) -> Result { + let response = self + .request(&Request::Prepare { + vm_id: vm_id.to_string(), + nic_index, + bridge: bridge.to_string(), + mac: mac.to_string(), + }) + .await?; + response.context("dstack-netd returned no attachment") + } + + pub(crate) async fn remove(&self, attachment: &NicAttachment) -> Result<()> { + self.request(&Request::Remove { + attachment: attachment.clone(), + }) + .await?; + Ok(()) + } + + pub(crate) async fn check(&self, attachment: &NicAttachment) -> Result<()> { + self.request(&Request::Check { + attachment: attachment.clone(), + }) + .await?; + Ok(()) + } + + async fn request(&self, request: &Request) -> Result> { + let mut stream = UnixStream::connect(&self.socket) + .await + .with_context(|| format!("failed to connect to {}", self.socket.display()))?; + let mut encoded = serde_json::to_vec(request)?; + if encoded.len() > MAX_REQUEST_BYTES { + bail!("dstack-netd request is too large"); + } + encoded.push(b'\n'); + stream.write_all(&encoded).await?; + stream.shutdown().await?; + + let mut response = String::new(); + AsyncBufReader::new(stream) + .read_line(&mut response) + .await + .context("failed to read dstack-netd response")?; + serde_json::from_str::(&response) + .context("invalid dstack-netd response")? + .into_result() + } +} + +pub(crate) async fn serve(cfg: CvmConfig) -> Result<()> { + let socket = &cfg.networking.netd_socket; + if socket.as_os_str().is_empty() { + bail!("cvm.networking.netd_socket must not be empty"); + } + if let Some(parent) = socket.parent() { + fs::create_dir_all(parent) + .with_context(|| format!("failed to create {}", parent.display()))?; + } + if socket.exists() { + if StdUnixStream::connect(socket).is_ok() { + bail!("dstack-netd is already listening on {}", socket.display()); + } + fs::remove_file(socket) + .with_context(|| format!("failed to remove stale {}", socket.display()))?; + } + let listener = UnixListener::bind(socket) + .with_context(|| format!("failed to bind {}", socket.display()))?; + if let Some(gid) = cfg.networking.netd_socket_gid { + chown(socket, None, Some(Gid::from_raw(gid)))?; + fs::set_permissions(socket, fs::Permissions::from_mode(0o660))?; + } else { + fs::set_permissions(socket, fs::Permissions::from_mode(0o600))?; + } + + let server = std::sync::Arc::new(Server::new(cfg)?); + loop { + let (stream, _) = listener.accept().await?; + let server = server.clone(); + tokio::spawn(async move { + if let Err(error) = handle_connection(stream, server).await { + tracing::warn!(%error, "dstack-netd request failed"); + } + }); + } +} + +async fn handle_connection(stream: UnixStream, server: std::sync::Arc) -> Result<()> { + let credentials = stream + .peer_cred() + .context("failed to read peer credentials")?; + if !server.allowed_uids.contains(&credentials.uid()) { + bail!( + "uid {} is not authorized for dstack-netd", + credentials.uid() + ); + } + + let (reader, mut writer) = stream.into_split(); + let mut request = Vec::new(); + AsyncBufReader::new(reader) + .take((MAX_REQUEST_BYTES + 1) as u64) + .read_until(b'\n', &mut request) + .await?; + if request.is_empty() || request.len() > MAX_REQUEST_BYTES || !request.ends_with(b"\n") { + bail!("invalid dstack-netd request framing"); + } + + let parsed = serde_json::from_slice::(&request); + let response = match parsed { + Ok(request) => { + let server = server.clone(); + match tokio::task::spawn_blocking(move || server.handle(request)).await { + Ok(Ok(attachment)) => Response::success(attachment), + Ok(Err(error)) => Response::error(format!("{error:#}")), + Err(error) => Response::error(format!("network worker failed: {error}")), + } + } + Err(error) => Response::error(format!("invalid request: {error}")), + }; + let mut encoded = serde_json::to_vec(&response)?; + encoded.push(b'\n'); + writer.write_all(&encoded).await?; + writer.shutdown().await?; + Ok(()) +} + +struct Server { + cfg: CvmConfig, + qemu_uid: u32, + allowed_uids: Vec, + operation_lock: std::sync::Mutex<()>, +} + +impl Server { + fn new(cfg: CvmConfig) -> Result { + let qemu_uid = if cfg.user.is_empty() { + 0 + } else { + User::from_name(&cfg.user) + .context("failed to resolve QEMU user")? + .with_context(|| format!("QEMU user '{}' does not exist", cfg.user))? + .uid + .as_raw() + }; + let allowed_uids = if cfg.networking.netd_allowed_uids.is_empty() { + vec![0] + } else { + cfg.networking.netd_allowed_uids.clone() + }; + Ok(Self { + cfg, + qemu_uid, + allowed_uids, + operation_lock: std::sync::Mutex::new(()), + }) + } + + fn handle(&self, request: Request) -> Result> { + let _guard = self + .operation_lock + .lock() + .map_err(|_| anyhow::anyhow!("dstack-netd operation lock is poisoned"))?; + match request { + Request::Prepare { + vm_id, + nic_index, + bridge, + mac, + } => self.prepare(&vm_id, nic_index, &bridge, &mac).map(Some), + Request::Remove { attachment } => { + self.validate_attachment_identity(&attachment)?; + self.remove_owned(&attachment)?; + Ok(None) + } + Request::Check { attachment } => { + self.validate_attachment_policy(&attachment)?; + self.check(&attachment)?; + Ok(None) + } + } + } + + fn prepare( + &self, + vm_id: &str, + nic_index: usize, + bridge: &str, + mac: &str, + ) -> Result { + validate_vm_id(vm_id)?; + validate_mac(mac)?; + let mut networking = self.cfg.networking.clone(); + networking.mode = NetworkingMode::Bridge; + networking.bridge = bridge.to_string(); + validate_resolved_network(&networking, &self.cfg.networking)?; + + let attachment = NicAttachment { + vm_id: vm_id.to_string(), + nic_index, + bridge: bridge.to_string(), + tap: tap_name(vm_id, nic_index)?, + mac: mac.to_ascii_lowercase(), + }; + self.validate_attachment_identity(&attachment)?; + + if Path::new("/sys/class/net").join(&attachment.tap).exists() { + self.verify_alias(&attachment)?; + self.remove_owned(&attachment)?; + } + + let result = self.create(&attachment); + if result.is_err() { + let _ = self.remove_owned(&attachment); + } + result.map(|()| attachment) + } + + fn create(&self, attachment: &NicAttachment) -> Result<()> { + let qemu_uid = self.qemu_uid.to_string(); + run( + "ip", + &[ + "tuntap", + "add", + "dev", + &attachment.tap, + "mode", + "tap", + "user", + &qemu_uid, + "vnet_hdr", + ], + )?; + run( + "ip", + &[ + "link", + "set", + "dev", + &attachment.tap, + "alias", + &tap_alias(attachment), + ], + )?; + run( + "ip", + &[ + "link", + "set", + "dev", + &attachment.tap, + "master", + &attachment.bridge, + ], + )?; + let mut bridge_args = vec![ + "link", + "set", + "dev", + &attachment.tap, + "learning", + "off", + "locked", + "on", + "guard", + "on", + "root_block", + "on", + ]; + if self.cfg.networking.isolate_bridge_ports { + bridge_args.extend(["isolated", "on"]); + } + run("bridge", &bridge_args)?; + run( + "bridge", + &[ + "fdb", + "replace", + &attachment.mac, + "dev", + &attachment.tap, + "master", + "static", + ], + )?; + run_with_stdin("nft", &["-f", "-"], &nft_rules(attachment))?; + run("ip", &["link", "set", "dev", &attachment.tap, "up"])?; + Ok(()) + } + + fn remove_owned(&self, attachment: &NicAttachment) -> Result<()> { + let path = Path::new("/sys/class/net").join(&attachment.tap); + if path.exists() { + self.verify_alias(attachment)?; + let _ = run("ip", &["link", "set", "dev", &attachment.tap, "down"]); + let _ = run("ip", &["link", "set", "dev", &attachment.tap, "nomaster"]); + } + let _ = run( + "nft", + &["delete", "table", "netdev", &nft_table_name(attachment)], + ); + if path.exists() { + run("ip", &["link", "delete", "dev", &attachment.tap])?; + } + Ok(()) + } + + fn check(&self, attachment: &NicAttachment) -> Result<()> { + self.verify_alias(attachment)?; + let master = fs::canonicalize( + Path::new("/sys/class/net") + .join(&attachment.tap) + .join("master"), + ) + .context("protected TAP has no bridge master")?; + if master.file_name().and_then(|name| name.to_str()) != Some(&attachment.bridge) { + bail!("protected TAP is attached to an unexpected bridge"); + } + let bridge_state = run_capture( + "bridge", + &["-details", "link", "show", "dev", &attachment.tap], + )?; + for required in ["learning off", "locked on", "guard on", "root_block on"] { + if !bridge_state.contains(required) { + bail!("protected TAP is missing bridge flag '{required}'"); + } + } + if self.cfg.networking.isolate_bridge_ports && !bridge_state.contains("isolated on") { + bail!("protected TAP is missing bridge isolation"); + } + + let fdb = run_capture("bridge", &["fdb", "show", "dev", &attachment.tap])?; + if !fdb.lines().any(|line| { + line.to_ascii_lowercase().starts_with(&attachment.mac) + && line.split_ascii_whitespace().any(|field| field == "static") + }) { + bail!("protected TAP is missing its static MAC entry"); + } + + let rules = run_capture( + "nft", + &["list", "table", "netdev", &nft_table_name(attachment)], + )?; + for required in [ + format!("device \"{}\"", attachment.tap), + format!("ether saddr != {}", attachment.mac), + "ether type ip6".to_string(), + ] { + if !rules.contains(&required) { + bail!("protected TAP nftables policy is incomplete"); + } + } + Ok(()) + } + + fn validate_attachment_identity(&self, attachment: &NicAttachment) -> Result<()> { + validate_vm_id(&attachment.vm_id)?; + validate_mac(&attachment.mac)?; + if attachment.tap != tap_name(&attachment.vm_id, attachment.nic_index)? { + bail!("attachment TAP name does not match its VM identity"); + } + Ok(()) + } + + fn validate_attachment_policy(&self, attachment: &NicAttachment) -> Result<()> { + self.validate_attachment_identity(attachment)?; + let mut networking = self.cfg.networking.clone(); + networking.mode = NetworkingMode::Bridge; + networking.bridge = attachment.bridge.clone(); + validate_resolved_network(&networking, &self.cfg.networking) + } + + fn verify_alias(&self, attachment: &NicAttachment) -> Result<()> { + let alias_path = Path::new("/sys/class/net") + .join(&attachment.tap) + .join("ifalias"); + let alias = fs::read_to_string(&alias_path) + .with_context(|| format!("failed to read {}", alias_path.display()))?; + if alias.trim() != tap_alias(attachment) { + bail!("refusing to modify foreign interface '{}'", attachment.tap); + } + Ok(()) + } +} + +fn validate_vm_id(vm_id: &str) -> Result<()> { + if vm_id.is_empty() + || vm_id.len() > 128 + || !vm_id + .chars() + .all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '-' | '_')) + { + bail!("invalid VM ID"); + } + Ok(()) +} + +fn validate_mac(mac: &str) -> Result<()> { + let bytes = mac + .split(':') + .map(|part| u8::from_str_radix(part, 16)) + .collect::, _>>() + .context("invalid MAC address")?; + if bytes.len() != 6 || bytes[0] & 1 != 0 || bytes.iter().all(|byte| *byte == 0) { + bail!("invalid unicast MAC address"); + } + Ok(()) +} + +fn tap_name(vm_id: &str, nic_index: usize) -> Result { + validate_vm_id(vm_id)?; + if nic_index > 99 { + bail!("NIC index exceeds the deterministic TAP naming range"); + } + let digest = hex::encode(Sha256::digest(vm_id.as_bytes())); + Ok(format!("dst{}n{nic_index}", &digest[..8])) +} + +fn tap_alias(attachment: &NicAttachment) -> String { + format!( + "{TAP_ALIAS_PREFIX}{}:{}:{}", + attachment.vm_id, attachment.nic_index, attachment.mac + ) +} + +fn nft_table_name(attachment: &NicAttachment) -> String { + let digest = hex::encode(Sha256::digest(attachment.vm_id.as_bytes())); + format!( + "{NFT_TABLE_PREFIX}{}_{}", + &digest[..8], + attachment.nic_index + ) +} + +fn nft_rules(attachment: &NicAttachment) -> String { + let table = nft_table_name(attachment); + format!( + r#"table netdev {table} {{ + chain ingress {{ + type filter hook ingress device "{tap}" priority -500; policy accept; + ether saddr != {mac} counter drop + ether type vlan counter drop + ether type 8021ad counter drop + ether type ip udp sport 67 counter drop + ether type ip accept + ether type arp accept + ether type ip6 counter drop + counter drop + }} +}} +"#, + tap = attachment.tap, + mac = attachment.mac, + ) +} + +fn run(program: &str, args: &[&str]) -> Result<()> { + run_capture(program, args).map(|_| ()) +} + +fn run_capture(program: &str, args: &[&str]) -> Result { + let output = Command::new(program) + .args(args) + .output() + .with_context(|| format!("failed to execute {program}"))?; + if !output.status.success() { + bail!( + "{} {} failed: {}", + program, + args.join(" "), + String::from_utf8_lossy(&output.stderr).trim() + ); + } + Ok(String::from_utf8_lossy(&output.stdout).into_owned()) +} + +fn run_with_stdin(program: &str, args: &[&str], input: &str) -> Result<()> { + let mut child = Command::new(program) + .args(args) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .with_context(|| format!("failed to execute {program}"))?; + child + .stdin + .take() + .context("missing child stdin")? + .write_all(input.as_bytes())?; + let output = child.wait_with_output()?; + if !output.status.success() { + bail!( + "{} {} failed: {}", + program, + args.join(" "), + String::from_utf8_lossy(&output.stderr).trim() + ); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn tap_and_table_names_are_deterministic_and_bounded() { + let tap = tap_name("123e4567-e89b-12d3-a456-426614174000", 12).unwrap(); + assert!(tap.starts_with("dst")); + assert!(tap.len() <= 15); + assert_eq!( + tap, + tap_name("123e4567-e89b-12d3-a456-426614174000", 12).unwrap() + ); + } + + #[test] + fn nft_policy_checks_mac_before_protocol_exceptions() { + let attachment = NicAttachment { + vm_id: "vm-1".into(), + nic_index: 0, + bridge: "br0".into(), + tap: tap_name("vm-1", 0).unwrap(), + mac: "02:aa:bb:cc:dd:ee".into(), + }; + let rules = nft_rules(&attachment); + assert!(rules.find("ether saddr !=").unwrap() < rules.find("udp sport 67").unwrap()); + assert!(rules.contains("ether type ip6 counter drop")); + } + + #[test] + fn attachment_identity_cannot_select_a_tap_name() { + let attachment = NicAttachment { + vm_id: "vm-1".into(), + nic_index: 0, + bridge: "br0".into(), + tap: "eth0".into(), + mac: "02:aa:bb:cc:dd:ee".into(), + }; + assert_ne!( + attachment.tap, + tap_name(&attachment.vm_id, attachment.nic_index).unwrap() + ); + } + + #[test] + fn rejects_multicast_and_malformed_mac_addresses() { + assert!(validate_mac("02:aa:bb:cc:dd:ee").is_ok()); + assert!(validate_mac("01:aa:bb:cc:dd:ee").is_err()); + assert!(validate_mac("not-a-mac").is_err()); + } +} diff --git a/dstack/vmm/vmm.toml b/dstack/vmm/vmm.toml index 940d50dd4..98daab9ed 100644 --- a/dstack/vmm/vmm.toml +++ b/dstack/vmm/vmm.toml @@ -114,6 +114,15 @@ restrict = false # Additional tenant-selectable bridges. The default bridge above is always # allowed; an empty list does not authorize arbitrary host bridges. # allowed_bridges = ["dstack-br1"] +# Enable deterministic TAP creation and host-enforced MAC/L2 filtering. Start +# the privileged backend with `dstack-vmm --config vmm.toml netd` first. +anti_spoof = false +netd_socket = "/run/dstack/netd.sock" +# Root is authorized when this list is empty. Add the unprivileged VMM UID and +# set the socket GID when the VMM service does not run as root. +netd_allowed_uids = [] +# netd_socket_gid = 1000 +isolate_bridge_ports = true [cvm.port_mapping] enabled = false