feat(vmm): enforce bridge MAC identity with netd

docs/bridge-networking.md

@@ -19,8 +19,23 @@ allowed_modes = ["user", "bridge"]
 bridge = "virbr0"
 # Optional additional bridges that VM requests may select.
 allowed_bridges = ["dstack-br1"]
+anti_spoof = true
+netd_socket = "/run/dstack/netd.sock"
+isolate_bridge_ports = true
 ```
 
+With `anti_spoof = true`, run the privileged backend before starting the VMM:
+
+```bash
+sudo dstack-vmm --config /etc/dstack/vmm.toml netd
+```
+
+The VMM fails closed if it cannot prepare a protected TAP. The backend creates
+a deterministic TAP, locks bridge learning to the assigned MAC, installs an
+nftables netdev-ingress policy, and returns the TAP name to QEMU. The default
+socket is root-only. For an unprivileged VMM, configure `netd_socket_gid` and
+`netd_allowed_uids`, and run both processes under matching service identities.
+
 ### Per-VM override
 
 Individual VMs can override the global networking mode via:

dstack/vmm/Cargo.toml

@@ -25,7 +25,7 @@ sha2.workspace = true
 hex.workspace = true
 fs-err.workspace = true
 getrandom = { workspace = true, features = ["std"] }
-nix = { workspace = true, features = ["user"] }
+nix = { workspace = true, features = ["fs", "user"] }
 dirs.workspace = true
 which.workspace = true
 clap = { workspace = true, features = ["derive", "string"] }

dstack/vmm/src/app.rs

@@ -30,6 +30,8 @@ use std::time::SystemTime;
 use supervisor_client::SupervisorClient;
 use tracing::{debug, error, info, warn};
 
+use crate::netd::{Client as NetdClient, NicAttachment};
+
 pub use image::{Image, ImageInfo};
 pub(crate) use network::{
     resolve_networking, resolved_networks, validate_resolved_network, validate_resolved_networks,
@@ -71,6 +73,23 @@ fn signal_pidfd(pid: u32, signal: libc::c_int) -> std::io::Result<()> {
     }
 }
 
+fn protected_attachment(
+    vm_id: &str,
+    networking: &Networking,
+    nic_index: usize,
+) -> Option<NicAttachment> {
+    if !networking.is_bridge() || !networking.anti_spoof || networking.tap.is_empty() {
+        return None;
+    }
+    Some(NicAttachment {
+        vm_id: vm_id.to_string(),
+        nic_index,
+        bridge: networking.bridge.clone(),
+        tap: networking.tap.clone(),
+        mac: network::mac_address_for_vm_index(vm_id, &networking.mac_prefix_bytes(), nic_index),
+    })
+}
+
 #[derive(Deserialize, Serialize, Debug, Clone)]
 pub struct PortMapping {
     pub address: IpAddr,
@@ -305,6 +324,71 @@ impl App {
         VmWorkDir::new(self.config.run_path.join(id))
     }
 
+    async fn prepare_protected_networks(
+        &self,
+        vm_id: &str,
+        networks: &mut [Networking],
+    ) -> Result<()> {
+        let client = NetdClient::new(&self.config.cvm.networking.netd_socket);
+        let mut prepared = Vec::new();
+        for (index, networking) in networks.iter_mut().enumerate() {
+            if !networking.is_bridge() || !networking.anti_spoof {
+                continue;
+            }
+            let mac =
+                network::mac_address_for_vm_index(vm_id, &networking.mac_prefix_bytes(), index);
+            match client.prepare(vm_id, index, &networking.bridge, &mac).await {
+                Ok(attachment) => {
+                    networking.tap = attachment.tap.clone();
+                    prepared.push(attachment);
+                }
+                Err(error) => {
+                    for attachment in prepared.iter().rev() {
+                        if let Err(cleanup_error) = client.remove(attachment).await {
+                            warn!(
+                                vm_id,
+                                tap = %attachment.tap,
+                                "failed to roll back protected NIC: {cleanup_error:#}"
+                            );
+                        }
+                    }
+                    return Err(error).context("failed to prepare protected bridge networking");
+                }
+            }
+        }
+        Ok(())
+    }
+
+    async fn remove_protected_networks(&self, vm_id: &str, networks: &[Networking]) {
+        let client = NetdClient::new(&self.config.cvm.networking.netd_socket);
+        for (index, networking) in networks.iter().enumerate().rev() {
+            let Some(attachment) = protected_attachment(vm_id, networking, index) else {
+                continue;
+            };
+            if let Err(error) = client.remove(&attachment).await {
+                warn!(
+                    vm_id = %attachment.vm_id,
+                    tap = %attachment.tap,
+                    "failed to remove protected NIC: {error:#}"
+                );
+            }
+        }
+    }
+
+    async fn check_protected_networks(&self, vm_id: &str, networks: &[Networking]) -> Result<()> {
+        let client = NetdClient::new(&self.config.cvm.networking.netd_socket);
+        for (index, networking) in networks.iter().enumerate() {
+            let Some(attachment) = protected_attachment(vm_id, networking, index) else {
+                continue;
+            };
+            client
+                .check(&attachment)
+                .await
+                .with_context(|| format!("protected NIC {} is not enforced", attachment.tap))?;
+        }
+        Ok(())
+    }
+
     pub fn new(config: Config, supervisor: SupervisorClient) -> Self {
         let cid_start = config.cvm.cid_start;
         let cid_end = cid_start.saturating_add(config.cvm.cid_pool_size);
@@ -424,16 +508,34 @@ impl App {
             append_boot_separator(&work_dir.stderr_file());
 
             let devices = self.try_allocate_gpus(&vm_config.manifest)?;
-            let processes = vm_config.config_qemu(&work_dir, &self.config.cvm, &devices)?;
-            let runtime_networks = resolved_networks(&vm_config.manifest, &self.config.cvm);
-            work_dir.set_runtime_networks(&runtime_networks)?;
+            let mut runtime_networks = resolved_networks(&vm_config.manifest, &self.config.cvm);
+            self.prepare_protected_networks(id, &mut runtime_networks)
+                .await?;
+            let processes = match vm_config.config_qemu_with_networks(
+                &work_dir,
+                &self.config.cvm,
+                &devices,
+                Some(&runtime_networks),
+            ) {
+                Ok(processes) => processes,
+                Err(error) => {
+                    self.remove_protected_networks(id, &runtime_networks).await;
+                    return Err(error);
+                }
+            };
+            if let Err(error) = work_dir.set_runtime_networks(&runtime_networks) {
+                self.remove_protected_networks(id, &runtime_networks).await;
+                return Err(error);
+            }
             {
                 let mut state = self.lock();
                 let vm_state = state.get_mut(id).context("VM not found")?;
-                vm_state.state.runtime_networks = runtime_networks;
+                vm_state.state.runtime_networks = runtime_networks.clone();
             }
             for process in processes {
                 if let Err(err) = self.supervisor.deploy(&process).await {
+                    let _ = self.supervisor.stop(id).await;
+                    self.remove_protected_networks(id, &runtime_networks).await;
                     if let Err(clear_err) = work_dir.clear_runtime_networks() {
                         warn!(
                             id,
@@ -465,6 +567,8 @@ impl App {
     pub async fn stop_vm(&self, id: &str) -> Result<()> {
         self.set_started(id, false)?;
         self.stop_vm_process(id).await?;
+        let networks = self.work_dir(id).runtime_networks();
+        self.remove_protected_networks(id, &networks).await;
         Ok(())
     }
 
@@ -574,6 +678,9 @@ impl App {
             }
         }
 
+        let runtime_networks = self.work_dir(id).runtime_networks();
+        self.remove_protected_networks(id, &runtime_networks).await;
+
         // Only delete the workdir for user-initiated removal or if .removing marker exists.
         // Orphaned supervisor processes without the marker keep their data intact.
         let vm_path = self.work_dir(id);
@@ -672,6 +779,22 @@ impl App {
             }
         }
 
+        // A protected guest must never continue running when its host-side
+        // enforcement cannot be proven after VMM restart.
+        for id in occupied_cids.keys() {
+            if !self.lock().vms.contains_key(id) {
+                continue;
+            }
+            let networks = self.work_dir(id).runtime_networks();
+            if let Err(error) = self.check_protected_networks(id, &networks).await {
+                error!(id, "protected networking reconciliation failed: {error:#}");
+                self.stop_vm_process(id)
+                    .await
+                    .with_context(|| format!("failed to quiesce VM {id}"))?;
+                self.remove_protected_networks(id, &networks).await;
+            }
+        }
+
         // Resume cleanup for VMs with .removing marker
         for id in removing_ids {
             self.spawn_finish_remove(&id);
@@ -1610,6 +1733,12 @@ mod tests {
             allowed_modes: vec![],
             bridge: "dstack-br0".to_string(),
             allowed_bridges: vec![],
+            anti_spoof: false,
+            netd_socket: Default::default(),
+            netd_allowed_uids: vec![],
+            netd_socket_gid: None,
+            isolate_bridge_ports: true,
+            tap: String::new(),
             mac_prefix: String::new(),
             net: String::new(),
             dhcp_start: String::new(),
@@ -1829,6 +1958,12 @@ mod tests {
             allowed_modes: vec![],
             bridge: "dstack-br0".to_string(),
             allowed_bridges: vec![],
+            anti_spoof: false,
+            netd_socket: Default::default(),
+            netd_allowed_uids: vec![],
+            netd_socket_gid: None,
+            isolate_bridge_ports: true,
+            tap: String::new(),
             mac_prefix: "02:aa:bb".to_string(),
             net: String::new(),
             dhcp_start: String::new(),

dstack/vmm/src/app/network.rs

@@ -159,6 +159,12 @@ mod tests {
             allowed_modes: vec![],
             bridge: bridge.to_string(),
             allowed_bridges: vec![],
+            anti_spoof: false,
+            netd_socket: Default::default(),
+            netd_allowed_uids: vec![],
+            netd_socket_gid: None,
+            isolate_bridge_ports: true,
+            tap: String::new(),
             mac_prefix: String::new(),
             net: String::new(),
             dhcp_start: String::new(),

dstack/vmm/src/app/qemu.rs

@@ -198,13 +198,16 @@ impl PreparedQemuLaunch {
         workdir: impl AsRef<Path>,
         cfg: &CvmConfig,
         gpus: &GpuConfig,
+        prepared_networks: Option<&[Networking]>,
     ) -> Result<Self> {
         let workdir = VmWorkDir::new(workdir);
         prepare_data_disk(vm, &workdir, cfg)?;
         prepare_shared_dir(&workdir)?;
         let app_compose = workdir.app_compose().context("failed to get app compose")?;
         let platform = cfg.resolved_platform();
-        let networks = resolved_networks(&vm.manifest, cfg);
+        let networks = prepared_networks
+            .map(<[Networking]>::to_vec)
+            .unwrap_or_else(|| resolved_networks(&vm.manifest, cfg));
         validate_resolved_networks(&networks, &cfg.networking)?;
         let volumes = vm
             .manifest
@@ -340,7 +343,17 @@ impl VmConfig {
         cfg: &CvmConfig,
         gpus: &GpuConfig,
     ) -> Result<Vec<ProcessConfig>> {
-        let prepared = PreparedQemuLaunch::prepare(self, workdir, cfg, gpus)?;
+        self.config_qemu_with_networks(workdir, cfg, gpus, None)
+    }
+
+    pub fn config_qemu_with_networks(
+        &self,
+        workdir: impl AsRef<Path>,
+        cfg: &CvmConfig,
+        gpus: &GpuConfig,
+        networks: Option<&[Networking]>,
+    ) -> Result<Vec<ProcessConfig>> {
+        let prepared = PreparedQemuLaunch::prepare(self, workdir, cfg, gpus, networks)?;
         let process = QemuCommandBuilder {
             vm: self,
             cfg,
@@ -600,7 +613,17 @@ impl QemuCommandBuilder<'_> {
                 }
                 NetworkingMode::Bridge => {
                     tracing::info!("bridge networking: mac={mac} bridge={}", networking.bridge);
-                    format!("bridge,id={net_id},br={}", networking.bridge)
+                    if networking.anti_spoof {
+                        if networking.tap.is_empty() {
+                            bail!("protected bridge networking is missing a prepared TAP");
+                        }
+                        format!(
+                            "tap,id={net_id},ifname={},script=no,downscript=no,vhost=off",
+                            networking.tap
+                        )
+                    } else {
+                        format!("bridge,id={net_id},br={}", networking.bridge)
+                    }
                 }
                 NetworkingMode::Custom => {
                     if !networking.netdev.contains(&format!("id={net_id}")) {

dstack/vmm/src/config.rs

@@ -523,6 +523,14 @@ pub enum NetworkingMode {
     Custom,
 }
 
+fn default_netd_socket() -> PathBuf {
+    PathBuf::from("/run/dstack/netd.sock")
+}
+
+fn default_true() -> bool {
+    true
+}
+
 /// Flat networking configuration. The `mode` field selects which backend is
 /// active; the remaining fields are only relevant for their respective mode
 /// and carry serde defaults so they can be omitted in the config file.
@@ -545,6 +553,31 @@ pub struct Networking {
     #[serde(default, skip_serializing)]
     pub allowed_bridges: Vec<String>,
 
+    /// Enable host-enforced MAC and L2 filtering through `dstack-netd`.
+    #[serde(default)]
+    pub anti_spoof: bool,
+
+    /// Unix socket exposed by the privileged networking backend.
+    #[serde(default = "default_netd_socket", skip_serializing)]
+    pub netd_socket: PathBuf,
+
+    /// UIDs authorized to call `dstack-netd`. Empty authorizes only root.
+    #[serde(default, skip_serializing)]
+    pub netd_allowed_uids: Vec<u32>,
+
+    /// Optional group owner for the netd socket. When set, the socket mode is
+    /// `0660`; otherwise it remains root-only `0600`.
+    #[serde(default, skip_serializing)]
+    pub netd_socket_gid: Option<u32>,
+
+    /// Prevent forwarding between protected bridge ports.
+    #[serde(default = "default_true")]
+    pub isolate_bridge_ports: bool,
+
+    /// Runtime-only deterministic TAP name returned by `dstack-netd`.
+    #[serde(default, skip_serializing_if = "String::is_empty")]
+    pub tap: String,
+
     // ── MAC prefix ─────────────────────────────────────────────────
     /// Fixed MAC address prefix (0-3 colon-separated hex bytes, e.g. "02:ab:cd").
     /// Remaining bytes are derived from the VM ID hash.

dstack/vmm/src/main.rs

@@ -28,6 +28,7 @@ mod guest_api_service;
 mod host_api_service;
 mod main_routes;
 mod main_service;
+mod netd;
 mod one_shot;
 mod openapi;
 mod vm_launcher;
@@ -60,6 +61,8 @@ enum Command {
     /// Internal per-VM QEMU/swtpm launcher.
     #[command(hide = true)]
     VmLauncher(VmLauncherArgs),
+    /// Run the privileged bridge networking backend.
+    Netd,
 }
 
 #[derive(ClapArgs)]
@@ -188,6 +191,7 @@ async fn main() -> Result<()> {
     // Handle commands
     match args.command.unwrap_or_default() {
         Command::VmLauncher(_) => unreachable!("launcher mode handled before config loading"),
+        Command::Netd => return netd::serve(config.cvm).await,
         Command::Run(run_args) => {
             // One-shot VM execution mode
             return one_shot::run_one_shot(

dstack/vmm/src/main_service.rs

@@ -341,6 +341,12 @@ fn networking_from_proto(proto: &rpc::NetworkingConfig) -> Result<Option<Network
         allowed_modes: vec![],
         bridge,
         allowed_bridges: vec![],
+        anti_spoof: false,
+        netd_socket: Default::default(),
+        netd_allowed_uids: vec![],
+        netd_socket_gid: None,
+        isolate_bridge_ports: true,
+        tap: String::new(),
         mac_prefix: String::new(),
         net: String::new(),
         dhcp_start: String::new(),