From 9a1353039f774cc324bad90f6c961efc450efe0b Mon Sep 17 00:00:00 2001 From: Igor Yamolov Date: Tue, 10 Nov 2026 12:27:01 +0100 Subject: [PATCH 01/22] SPIKE: TREL-like IP backbone for satellite routers (#84) Throwaway exploration, not for merge: - ziggurat-phy-trel: RadioPhy wrapper carrying 802.15.4 frames over UDP between Ziggurat nodes, mDNS (_zigtrel._udp) / static / learned peers, LQI pinning for backbone peers, and a test-only deny list - server: --backbone-* flags, --satellite-of to start in router role from the coordinator's get_network_info, periodic [diag] logging Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 77 ++- crates/ziggurat-phy-trel/Cargo.toml | 21 + crates/ziggurat-phy-trel/SPIKE.md | 59 ++ crates/ziggurat-phy-trel/src/lib.rs | 753 +++++++++++++++++++++ crates/ziggurat-phy-trel/tests/loopback.rs | 196 ++++++ crates/ziggurat-server/Cargo.toml | 3 +- crates/ziggurat-server/src/main.rs | 136 +++- crates/ziggurat-server/src/satellite.rs | 112 +++ 8 files changed, 1338 insertions(+), 19 deletions(-) create mode 100644 crates/ziggurat-phy-trel/Cargo.toml create mode 100644 crates/ziggurat-phy-trel/SPIKE.md create mode 100644 crates/ziggurat-phy-trel/src/lib.rs create mode 100644 crates/ziggurat-phy-trel/tests/loopback.rs create mode 100644 crates/ziggurat-server/src/satellite.rs diff --git a/Cargo.lock b/Cargo.lock index 331cd00..c099752 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -637,12 +637,29 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + [[package]] name = "find-msvc-tools" version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" +[[package]] +name = "flume" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e139bc46ca777eb5efaf62df0ab8cc5fd400866427e56c68b22e414e53bd3be" +dependencies = [ + "futures-core", + "futures-sink", + "spin 0.9.9", +] + [[package]] name = "fnv" version = "1.0.7" @@ -841,6 +858,16 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" +[[package]] +name = "if-addrs" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0a05c691e1fae256cf7013d99dad472dc52d5543322761f83ec8d47eab40d2b" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + [[package]] name = "indexmap" version = "2.14.2" @@ -957,6 +984,21 @@ dependencies = [ "regex-automata", ] +[[package]] +name = "mdns-sd" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6743ed743ff2bada2b8651ef298d3f98addb0b74844f19cdd57137d8bd0219e" +dependencies = [ + "fastrand", + "flume", + "if-addrs", + "log", + "mio", + "socket-pktinfo", + "socket2", +] + [[package]] name = "memchr" version = "2.8.3" @@ -1316,6 +1358,17 @@ version = "1.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9395f0f0eee849a9b707b2f06bb92a6a422090e2123bb2ef8e87a0e61892a8e" +[[package]] +name = "socket-pktinfo" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "612942246d0cc239cfd83af1dfd39be47f649208a3524e5e9da651910128e0ac" +dependencies = [ + "libc", + "socket2", + "windows-sys 0.61.2", +] + [[package]] name = "socket2" version = "0.6.5" @@ -1326,6 +1379,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "spin" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" +dependencies = [ + "lock_api", +] + [[package]] name = "spin" version = "0.12.3" @@ -1860,7 +1922,7 @@ dependencies = [ "futures", "parking_lot", "rand", - "spin", + "spin 0.12.3", "thiserror 2.0.21", "tokio", "tracing", @@ -1901,6 +1963,18 @@ dependencies = [ "ziggurat-spinel", ] +[[package]] +name = "ziggurat-phy-trel" +version = "0.1.0" +dependencies = [ + "mdns-sd", + "parking_lot", + "tokio", + "tracing", + "ziggurat-ieee-802154", + "ziggurat-phy", +] + [[package]] name = "ziggurat-protocol" version = "0.1.0" @@ -1930,6 +2004,7 @@ dependencies = [ "ziggurat-driver", "ziggurat-phy", "ziggurat-phy-spinel", + "ziggurat-phy-trel", "ziggurat-protocol", "ziggurat-spinel", "ziggurat-zigbee", diff --git a/crates/ziggurat-phy-trel/Cargo.toml b/crates/ziggurat-phy-trel/Cargo.toml new file mode 100644 index 0000000..22c684d --- /dev/null +++ b/crates/ziggurat-phy-trel/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "ziggurat-phy-trel" +version.workspace = true +description = "SPIKE: TREL-like IP backbone link wrapped around a ziggurat-phy RadioPhy" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +authors.workspace = true +repository.workspace = true + +[dependencies] +ziggurat-ieee-802154.workspace = true +ziggurat-phy.workspace = true + +parking_lot = "0.12.5" +tokio = { version = "1.53.1", features = ["rt", "time", "sync", "net", "macros"] } +tracing = "0.1" +mdns-sd = "0.21.7" + +[dev-dependencies] +tokio = { version = "1.53.1", features = ["rt-multi-thread", "macros", "time", "sync", "net"] } diff --git a/crates/ziggurat-phy-trel/SPIKE.md b/crates/ziggurat-phy-trel/SPIKE.md new file mode 100644 index 0000000..6af7797 --- /dev/null +++ b/crates/ziggurat-phy-trel/SPIKE.md @@ -0,0 +1,59 @@ +# SPIKE: TREL-like IP backbone for satellite routers (zigpy/ziggurat#84) + +Throwaway. Answers: can Ziggurat routers, configured out-of-band and linked to the +coordinator over IP, extend coverage without any NWK changes? + +## How it works + +- `TrelPhy

` wraps any `RadioPhy`. The backbone is a second "radio" that only + reaches other Ziggurat nodes: raw 802.15.4 frames in UDP datagrams. +- Unicast to a backbone peer → UDP only, ack + 3×50 ms retries, falls back to the air. + Broadcast → air **and** every peer (NWK broadcast dedup absorbs duplicates). +- Frames from peers are reported with LQI 255 (on either link), so NWK link cost to a + peer is 1 and routing prefers the backbone. +- Peers: mDNS `_zigtrel._udp` (TXT `eui64`, `nwk`, `pan`, `v`), `--backbone-peer`, + or learned from the first datagram. Different PAN → ignored. +- Satellite: `--satellite-of ws://coord:9999` fetches `get_network_info` from the + coordinator's Ziggurat API and starts in router role. NWK address derived from the + EUI64 (or `--satellite-nwk`). Frame counters seeded from the clock. +- `--deny ` fakes "out of radio range" (drops air frames from it, refuses + air unicasts to it). +- Every 30 s with the backbone on: `[diag]` log lines with peers and router neighbours + (LQA, incoming/outgoing cost). + +## Bench + +Coordinator (ZHA connects here as usual): + +```bash +cargo run --bin ziggurat -- --device /dev/serial/by-id/ --listen 0.0.0.0:9999 \ + --backbone-port 19790 +``` + +Satellite (same box is fine, different port/radio): + +```bash +cargo run --bin ziggurat -- --device /dev/serial/by-id/ --listen 127.0.0.1:9998 \ + --backbone-port 19791 --satellite-of ws://127.0.0.1:9999 +``` + +Experiment: + +1. Both up → `[diag]` shows each other as backbone peer, neighbour cost 1/1. +2. Pair an IKEA outlet via ZHA normally (through the coordinator). +3. Restart the coordinator with `--deny --deny `. +4. Toggle the outlet from ZHA; it must route coordinator → satellite (backbone) → + outlet, and back. Watch `[diag]` on both and `Mgmt_Lqi` of the outlet. +5. Drop the deny list; observe which routes win and whether real routers' + Link Status handling of two "coordinator-ish" neighbours causes trouble. + +Loopback smoke tests (no hardware): `cargo test -p ziggurat-phy-trel`. + +## Known gaps (by design for a spike) + +- **Satellites can't join devices.** Router-side join (APS Update Device to the TC, + unwrapping Tunnel commands) isn't implemented; pair through the coordinator first. +- **Deny list is approximate.** The RCP still hardware-ACKs frames from a denied + device. Fine once routes are established; misleading for direct joins. +- No peer authentication (NWK frames are encrypted, but a LAN host could flood or + replay). IPv4 only. No children on satellites. Satellite isn't announced to ZHA. diff --git a/crates/ziggurat-phy-trel/src/lib.rs b/crates/ziggurat-phy-trel/src/lib.rs new file mode 100644 index 0000000..13ac178 --- /dev/null +++ b/crates/ziggurat-phy-trel/src/lib.rs @@ -0,0 +1,753 @@ +//! SPIKE (throwaway): a TREL-like IP backbone, wrapped around any [`RadioPhy`]. +//! +//! The backbone behaves as a second 802.15.4 "radio" that only reaches other Ziggurat +//! nodes. Raw 802.15.4 frames (FCS stripped) are carried in UDP datagrams: +//! +//! * unicasts whose MAC destination is a known backbone peer go over UDP only, with a +//! tiny ack/retransmit standing in for the MAC ACK, falling back to the air on +//! failure; +//! * broadcasts go over the air *and* to every backbone peer (NWK broadcast dedup +//! absorbs the duplicates); +//! * frames from backbone peers are reported with LQI 255 (whichever link they came in +//! on), so NWK link costs to backbone peers are 1 and routing prefers the backbone +//! without any NWK changes. +//! +//! Peers are discovered with mDNS/DNS-SD (`_zigtrel._udp`), configured statically, or +//! learned from the first datagram they send us. +//! +//! A test-only deny list fakes "out of radio range": over-the-air frames whose MAC +//! source is denied are dropped, and over-the-air unicasts to a denied MAC destination +//! report `NoAck` without being transmitted. The RCP still auto-ACKs frames from denied +//! devices at the hardware level; see the spike notes. + +use std::collections::{HashMap, HashSet, VecDeque}; +use std::net::{IpAddr, SocketAddr}; +use std::sync::Arc; +use std::sync::atomic::{AtomicU16, Ordering}; +use std::time::{Duration, Instant}; + +use parking_lot::Mutex; +use tokio::net::UdpSocket; +use tokio::sync::{mpsc, oneshot}; +use ziggurat_ieee_802154::types::{Eui64, Nwk, PanId}; +use ziggurat_phy::{RadioConfig, RadioError, RadioPhy, Receiver, RxFrame, TxFrame, TxResult}; + +pub const SERVICE_TYPE: &str = "_zigtrel._udp.local."; +const MAGIC: [u8; 2] = *b"ZT"; +const VERSION: u8 = 1; +const HEADER_LEN: usize = 19; +const KIND_DATA: u8 = 0; +const KIND_ACK: u8 = 1; +const FLAG_ACK_REQUEST: u8 = 0x01; + +const ACK_TIMEOUT: Duration = Duration::from_millis(50); +const ATTEMPTS: usize = 3; +const DEDUP_WINDOW: usize = 64; +/// LQI and RSSI reported for frames from backbone peers. +const BACKBONE_LQI: u8 = 255; +const BACKBONE_RSSI: i8 = -20; + +#[derive(Debug, Clone, Default)] +pub struct DenyList { + pub eui64: HashSet, + pub nwk: HashSet, +} + +impl DenyList { + fn denies(&self, address: &MacAddress) -> bool { + match address { + MacAddress::Short(nwk) => self.nwk.contains(nwk), + MacAddress::Extended(eui64) => self.eui64.contains(eui64), + MacAddress::None => false, + } + } +} + +#[derive(Debug, Clone)] +pub struct TrelConfig { + /// UDP port to listen on (0 disables the backbone entirely) + pub port: u16, + /// Peers reachable without discovery; their identity is learned from traffic + pub static_peers: Vec, + pub mdns: bool, + pub deny: DenyList, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum PeerSource { + Mdns, + Learned, +} + +#[derive(Debug, Clone)] +struct Peer { + addr: SocketAddr, + nwk: Option, + source: PeerSource, + last_rx: Option, +} + +#[derive(Debug, Clone, Copy)] +struct Identity { + eui64: Eui64, + nwk: Nwk, + pan_id: PanId, + channel: u8, +} + +#[derive(Default)] +struct Shared { + identity: Option, + peers: HashMap, + /// Recently delivered (sender, backbone seq), to drop retransmits of a frame whose + /// ack was lost. + seen: HashMap>, + rx_tx: Option>, + mdns_fullname: Option, +} + +struct Backbone { + socket: Arc, + shared: Arc>, + pending: Arc>>>, + seq: AtomicU16, + static_peers: Vec, + mdns: Option, + port: u16, +} + +pub struct TrelPhy { + inner: P, + backbone: Option, + deny: DenyList, + shared: Arc>, +} + +/// The merged received-frame stream. +pub struct TrelRx(mpsc::Receiver); + +impl Receiver for TrelRx { + async fn recv(&mut self) -> Option { + self.0.recv().await + } +} + +impl TrelPhy

{ + /// Must be called from within a Tokio runtime. + pub async fn new(inner: P, config: TrelConfig) -> std::io::Result { + let shared = Arc::new(Mutex::new(Shared::default())); + + let backbone = if config.port == 0 { + None + } else { + let socket = Arc::new(UdpSocket::bind(("0.0.0.0", config.port)).await?); + tracing::info!("TREL backbone listening on UDP {}", config.port); + + let mdns = if config.mdns { + match mdns_sd::ServiceDaemon::new() { + Ok(daemon) => Some(daemon), + Err(e) => { + tracing::warn!("TREL: mDNS unavailable, static peers only: {e}"); + None + } + } + } else { + None + }; + + let backbone = Backbone { + socket, + shared: Arc::clone(&shared), + pending: Arc::new(Mutex::new(HashMap::new())), + seq: AtomicU16::new(rand_u16()), + static_peers: config.static_peers, + mdns, + port: config.port, + }; + backbone.spawn_udp_reader(); + backbone.spawn_mdns_browser(); + Some(backbone) + }; + + Ok(Self { + inner, + backbone, + deny: config.deny, + shared, + }) + } + + pub const fn inner(&self) -> &P { + &self.inner + } +} + +impl Backbone { + fn spawn_udp_reader(&self) { + let socket = Arc::clone(&self.socket); + let shared = Arc::clone(&self.shared); + let pending = Arc::clone(&self.pending); + + tokio::spawn(async move { + let mut buf = vec![0u8; 2048]; + loop { + let (len, from) = match socket.recv_from(&mut buf).await { + Ok(r) => r, + Err(e) => { + tracing::warn!("TREL: UDP receive failed: {e}"); + tokio::time::sleep(Duration::from_millis(100)).await; + continue; + } + }; + let Some(header) = WireHeader::parse(&buf[..len]) else { + tracing::debug!("TREL: dropping malformed datagram from {from}"); + continue; + }; + + if header.kind == KIND_ACK { + if let Some(waiter) = pending.lock().remove(&header.seq) { + let _ = waiter.send(()); + } + continue; + } + + let psdu = buf[HEADER_LEN..len].to_vec(); + let (deliver, ack_to) = { + let mut shared = shared.lock(); + let Some(identity) = shared.identity else { + continue; + }; + if header.pan_id != identity.pan_id || header.eui64 == identity.eui64 { + continue; + } + + // Learn (or refresh) the peer from its own traffic + let peer = shared.peers.entry(header.eui64).or_insert_with(|| { + tracing::info!( + "TREL: learned peer {:?} / {:?} at {from}", + header.eui64, + header.nwk + ); + Peer { + addr: from, + nwk: None, + source: PeerSource::Learned, + last_rx: None, + } + }); + peer.addr = from; + peer.nwk = Some(header.nwk); + peer.last_rx = Some(Instant::now()); + + let seen = shared.seen.entry(header.eui64).or_default(); + let duplicate = seen.contains(&header.seq); + if !duplicate { + seen.push_back(header.seq); + if seen.len() > DEDUP_WINDOW { + seen.pop_front(); + } + } + + let ack_to = (header.flags & FLAG_ACK_REQUEST != 0).then_some(from); + let deliver = (!duplicate).then(|| { + ( + shared.rx_tx.clone(), + RxFrame { + psdu, + channel: identity.channel, + rssi: BACKBONE_RSSI, + lqi: BACKBONE_LQI, + timestamp_us: 0, + }, + ) + }); + (deliver, ack_to.map(|to| (to, identity))) + }; + + if let Some((to, identity)) = ack_to { + let ack = WireHeader { + kind: KIND_ACK, + flags: 0, + seq: header.seq, + eui64: identity.eui64, + nwk: identity.nwk, + pan_id: identity.pan_id, + } + .to_bytes(); + let _ = socket.send_to(&ack, to).await; + } + + if let Some((Some(tx), frame)) = deliver { + tracing::trace!( + "TREL: rx {} bytes from {:?}", + frame.psdu.len(), + header.eui64 + ); + let _ = tx.try_send(frame); + } + } + }); + } + + fn spawn_mdns_browser(&self) { + let Some(daemon) = &self.mdns else { return }; + let events = match daemon.browse(SERVICE_TYPE) { + Ok(events) => events, + Err(e) => { + tracing::warn!("TREL: mDNS browse failed: {e}"); + return; + } + }; + let shared = Arc::clone(&self.shared); + + tokio::spawn(async move { + while let Ok(event) = events.recv_async().await { + match event { + mdns_sd::ServiceEvent::ServiceResolved(info) => { + let Some(eui64) = info + .get_property_val_str("eui64") + .and_then(|s| Eui64::try_from_hex(s).ok()) + else { + continue; + }; + let pan_id = info + .get_property_val_str("pan") + .and_then(|s| PanId::try_from_hex(s).ok()); + let nwk = info + .get_property_val_str("nwk") + .and_then(|s| Nwk::try_from_hex(s).ok()); + let Some(ip) = info.get_addresses_v4().into_iter().next() else { + continue; + }; + let addr = SocketAddr::new(IpAddr::V4(ip), info.get_port()); + + let mut shared = shared.lock(); + let Some(identity) = shared.identity else { + continue; + }; + if eui64 == identity.eui64 || pan_id != Some(identity.pan_id) { + continue; + } + tracing::info!("TREL: mDNS peer {eui64:?} / {nwk:?} at {addr}"); + let peer = shared.peers.entry(eui64).or_insert(Peer { + addr, + nwk, + source: PeerSource::Mdns, + last_rx: None, + }); + peer.addr = addr; + if nwk.is_some() { + peer.nwk = nwk; + } + } + mdns_sd::ServiceEvent::ServiceRemoved(_, fullname) => { + let Some(eui64) = fullname + .split('.') + .next() + .and_then(|s| Eui64::try_from_hex(s).ok()) + else { + continue; + }; + let mut shared = shared.lock(); + if shared + .peers + .get(&eui64) + .is_some_and(|p| p.source == PeerSource::Mdns) + { + tracing::info!("TREL: mDNS peer {eui64:?} went away"); + shared.peers.remove(&eui64); + } + } + _ => {} + } + } + }); + } + + fn advertise(&self, identity: Identity) { + let Some(daemon) = &self.mdns else { return }; + let mut shared = self.shared.lock(); + if let Some(old) = shared.mdns_fullname.take() { + let _ = daemon.unregister(&old); + } + + let instance = hex_be(&identity.eui64.to_be_bytes()); + let properties: HashMap = [ + ("v".to_string(), VERSION.to_string()), + ("eui64".to_string(), instance.clone()), + ("nwk".to_string(), format!("{:04x}", identity.nwk.as_u16())), + ("pan".to_string(), format!("{:04x}", identity.pan_id.0)), + ] + .into_iter() + .collect(); + + let host = format!("ziggurat-{instance}.local."); + match mdns_sd::ServiceInfo::new(SERVICE_TYPE, &instance, &host, "", self.port, properties) { + Ok(info) => { + let info = info.enable_addr_auto(); + let fullname = info.get_fullname().to_string(); + match daemon.register(info) { + Ok(()) => { + tracing::info!("TREL: advertising {fullname}"); + shared.mdns_fullname = Some(fullname); + } + Err(e) => tracing::warn!("TREL: mDNS register failed: {e}"), + } + } + Err(e) => tracing::warn!("TREL: bad mDNS service info: {e}"), + } + } + + /// Every address a broadcast goes to: known peers plus unidentified static peers. + fn broadcast_targets(&self) -> Vec { + let shared = self.shared.lock(); + let mut targets: Vec = shared.peers.values().map(|p| p.addr).collect(); + for addr in &self.static_peers { + if !targets.contains(addr) { + targets.push(*addr); + } + } + targets + } + + fn peer_for(&self, dest: &MacAddress) -> Option { + let shared = self.shared.lock(); + match dest { + MacAddress::Extended(eui64) => shared.peers.get(eui64).map(|p| p.addr), + MacAddress::Short(nwk) => shared + .peers + .values() + .find(|p| p.nwk == Some(*nwk)) + .map(|p| p.addr), + MacAddress::None => None, + } + } + + fn header(&self, kind: u8, flags: u8, seq: u16) -> Option> { + let identity = self.shared.lock().identity?; + Some( + WireHeader { + kind, + flags, + seq, + eui64: identity.eui64, + nwk: identity.nwk, + pan_id: identity.pan_id, + } + .to_bytes(), + ) + } + + async fn send_broadcast(&self, psdu: &[u8]) { + let seq = self.seq.fetch_add(1, Ordering::Relaxed); + let Some(mut datagram) = self.header(KIND_DATA, 0, seq) else { + return; + }; + datagram.extend_from_slice(psdu); + for target in self.broadcast_targets() { + let _ = self.socket.send_to(&datagram, target).await; + } + } + + /// Reliable-ish unicast to one peer. `true` once acked. + async fn send_unicast(&self, to: SocketAddr, psdu: &[u8], ack_request: bool) -> bool { + let seq = self.seq.fetch_add(1, Ordering::Relaxed); + let flags = if ack_request { FLAG_ACK_REQUEST } else { 0 }; + let Some(mut datagram) = self.header(KIND_DATA, flags, seq) else { + return false; + }; + datagram.extend_from_slice(psdu); + + if !ack_request { + return self.socket.send_to(&datagram, to).await.is_ok(); + } + + for attempt in 0..ATTEMPTS { + let (tx, rx) = oneshot::channel(); + self.pending.lock().insert(seq, tx); + if self.socket.send_to(&datagram, to).await.is_err() { + break; + } + if tokio::time::timeout(ACK_TIMEOUT, rx).await.is_ok() { + if attempt > 0 { + tracing::debug!("TREL: unicast to {to} acked after {} retries", attempt); + } + return true; + } + } + self.pending.lock().remove(&seq); + tracing::debug!("TREL: unicast to {to} not acked"); + false + } +} + +impl RadioPhy for TrelPhy

{ + type Exclusive<'a> = P::Exclusive<'a>; + type RxStream = TrelRx; + type ResetStream = P::ResetStream; + + async fn reset(&self) -> Result<(), RadioError> { + self.inner.reset().await + } + + async fn reconfigure(&self, config: &RadioConfig) -> Result<(), RadioError> { + self.inner.reconfigure(config).await?; + + // The capture config uses a broadcast address; there is no identity to announce + if config.short_address == Nwk(0xFFFF) || config.promiscuous { + return Ok(()); + } + let identity = Identity { + eui64: config.extended_address, + nwk: config.short_address, + pan_id: config.pan_id, + channel: config.channel, + }; + let changed = { + let mut shared = self.shared.lock(); + let changed = shared.identity.is_none_or(|old| { + old.eui64 != identity.eui64 + || old.nwk != identity.nwk + || old.pan_id != identity.pan_id + }); + shared.identity = Some(identity); + changed + }; + if changed && let Some(backbone) = &self.backbone { + backbone.advertise(identity); + } + Ok(()) + } + + async fn set_frame_pending_table( + &self, + short: &[Nwk], + extended: &[Eui64], + ) -> Result<(), RadioError> { + self.inner.set_frame_pending_table(short, extended).await + } + + async fn transmit(&self, frame: TxFrame) -> Result { + let Some(mac) = MacHeader::parse(&frame.psdu) else { + return self.inner.transmit(frame).await; + }; + + if let Some(backbone) = &self.backbone { + if mac.dest.is_broadcast() { + backbone.send_broadcast(&frame.psdu).await; + } else if let Some(peer) = backbone.peer_for(&mac.dest) { + if backbone + .send_unicast(peer, &frame.psdu, mac.ack_request) + .await + { + tracing::trace!("TREL: tx to {:?} via backbone", mac.dest); + return Ok(TxResult::Acked); + } + tracing::info!( + "TREL: backbone to {:?} failed, falling back to air", + mac.dest + ); + } + } + + if self.deny.denies(&mac.dest) { + tracing::trace!("TREL deny: not transmitting to {:?}", mac.dest); + return Ok(if mac.ack_request { + TxResult::NoAck + } else { + TxResult::Acked + }); + } + + self.inner.transmit(frame).await + } + + async fn energy_detect(&self, channel: u8, duration: Duration) -> Result { + self.inner.energy_detect(channel, duration).await + } + + async fn lock(&self) -> Self::Exclusive<'_> { + self.inner.lock().await + } + + fn subscribe_rx(&self) -> TrelRx { + let (tx, rx) = mpsc::channel(64); + self.shared.lock().rx_tx = Some(tx.clone()); + + let mut inner_rx = self.inner.subscribe_rx(); + let deny = self.deny.clone(); + let shared = Arc::clone(&self.shared); + let has_backbone = self.backbone.is_some(); + + // Over-the-air frames: apply the deny list and pin peers' LQI + tokio::spawn(async move { + while let Some(mut frame) = inner_rx.recv().await { + if let Some(mac) = MacHeader::parse(&frame.psdu) { + if deny.denies(&mac.source) { + tracing::trace!("TREL deny: dropping air frame from {:?}", mac.source); + continue; + } + if has_backbone { + let is_peer = { + let shared = shared.lock(); + match &mac.source { + MacAddress::Extended(eui64) => shared.peers.contains_key(eui64), + MacAddress::Short(nwk) => { + shared.peers.values().any(|p| p.nwk == Some(*nwk)) + } + MacAddress::None => false, + } + }; + if is_peer { + frame.lqi = BACKBONE_LQI; + frame.rssi = BACKBONE_RSSI; + } + } + } + if tx.send(frame).await.is_err() { + break; + } + } + }); + + TrelRx(rx) + } + + fn subscribe_reset(&self) -> Self::ResetStream { + self.inner.subscribe_reset() + } +} + +impl TrelPhy

{ + /// Peers currently known, for diagnostics. + pub fn peers(&self) -> Vec { + self.shared + .lock() + .peers + .iter() + .map(|(eui64, p)| { + format!( + "{eui64:?} nwk={:?} addr={} via={:?} last_rx={:?}", + p.nwk, + p.addr, + p.source, + p.last_rx.map(|t| t.elapsed()) + ) + }) + .collect() + } +} + +// -- wire format --------------------------------------------------------------------- + +/// `"ZT" | ver | kind | flags | seq:u16le | eui64 (wire order) | nwk:u16le | pan:u16le` +struct WireHeader { + kind: u8, + flags: u8, + seq: u16, + eui64: Eui64, + nwk: Nwk, + pan_id: PanId, +} + +impl WireHeader { + fn parse(bytes: &[u8]) -> Option { + if bytes.len() < HEADER_LEN || bytes[..2] != MAGIC || bytes[2] != VERSION { + return None; + } + Some(Self { + kind: bytes[3], + flags: bytes[4], + seq: u16::from_le_bytes([bytes[5], bytes[6]]), + eui64: Eui64(bytes[7..15].try_into().ok()?), + nwk: Nwk(u16::from_le_bytes([bytes[15], bytes[16]])), + pan_id: PanId(u16::from_le_bytes([bytes[17], bytes[18]])), + }) + } + + fn to_bytes(&self) -> Vec { + let mut out = Vec::with_capacity(HEADER_LEN + 128); + out.extend_from_slice(&MAGIC); + out.push(VERSION); + out.push(self.kind); + out.push(self.flags); + out.extend_from_slice(&self.seq.to_le_bytes()); + out.extend_from_slice(&self.eui64.0); + out.extend_from_slice(&self.nwk.0.to_le_bytes()); + out.extend_from_slice(&self.pan_id.0.to_le_bytes()); + out + } +} + +// -- minimal 802.15.4 MAC header parsing -------------------------------------------- + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum MacAddress { + None, + Short(Nwk), + Extended(Eui64), +} + +impl MacAddress { + const fn is_broadcast(&self) -> bool { + matches!(self, Self::Short(Nwk(0xFFFF))) + } +} + +struct MacHeader { + ack_request: bool, + dest: MacAddress, + source: MacAddress, +} + +impl MacHeader { + fn parse(psdu: &[u8]) -> Option { + let fcf = u16::from_le_bytes([*psdu.first()?, *psdu.get(1)?]); + let ack_request = fcf & (1 << 5) != 0; + let pan_compression = fcf & (1 << 6) != 0; + let dest_mode = (fcf >> 10) & 0b11; + let src_mode = (fcf >> 14) & 0b11; + + let mut i = 3; // FCF + sequence number + let mut read_addr = |mode: u16, with_pan: bool| -> Option { + if mode == 0 { + return Some(MacAddress::None); + } + if with_pan { + i += 2; + } + match mode { + 2 => { + let nwk = u16::from_le_bytes([*psdu.get(i)?, *psdu.get(i + 1)?]); + i += 2; + Some(MacAddress::Short(Nwk(nwk))) + } + 3 => { + let eui64 = Eui64(psdu.get(i..i + 8)?.try_into().ok()?); + i += 8; + Some(MacAddress::Extended(eui64)) + } + _ => None, + } + }; + let dest = read_addr(dest_mode, true)?; + let source = read_addr(src_mode, !pan_compression)?; + Some(Self { + ack_request, + dest, + source, + }) + } +} + +fn hex_be(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} + +fn rand_u16() -> u16 { + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.subsec_nanos()) + .unwrap_or(0); + (nanos ^ (nanos >> 16)) as u16 +} diff --git a/crates/ziggurat-phy-trel/tests/loopback.rs b/crates/ziggurat-phy-trel/tests/loopback.rs new file mode 100644 index 0000000..7ec9b8b --- /dev/null +++ b/crates/ziggurat-phy-trel/tests/loopback.rs @@ -0,0 +1,196 @@ +//! SPIKE smoke test: two backbone nodes over loopback, each wrapping a fake radio. + +use std::sync::Arc; +use std::time::Duration; + +use parking_lot::Mutex; +use tokio::sync::mpsc; +use ziggurat_ieee_802154::types::{Eui64, Nwk, PanId}; +use ziggurat_phy::*; +use ziggurat_phy_trel::{DenyList, TrelConfig, TrelPhy}; + +#[derive(Default)] +struct FakeState { + sent: Mutex>>, + rx: Mutex>>, +} + +struct FakeRadio(Arc); + +struct Rx(mpsc::Receiver); +impl Receiver for Rx { + async fn recv(&mut self) -> Option { + self.0.recv().await + } +} +struct NoExclusive; +impl ExclusiveRadio for NoExclusive { + async fn set_channel(&self, _: u8) -> Result<(), RadioError> { + Ok(()) + } + async fn transmit(&self, _: TxFrame) -> Result { + Ok(TxResult::Acked) + } +} + +impl RadioPhy for FakeRadio { + type Exclusive<'a> = NoExclusive; + type RxStream = Rx; + type ResetStream = Rx; + async fn reset(&self) -> Result<(), RadioError> { + Ok(()) + } + async fn reconfigure(&self, _: &RadioConfig) -> Result<(), RadioError> { + Ok(()) + } + async fn set_frame_pending_table(&self, _: &[Nwk], _: &[Eui64]) -> Result<(), RadioError> { + Ok(()) + } + async fn transmit(&self, frame: TxFrame) -> Result { + self.0.sent.lock().push(frame.psdu); + Ok(TxResult::NoAck) + } + async fn energy_detect(&self, _: u8, _: Duration) -> Result { + Ok(0) + } + async fn lock(&self) -> NoExclusive { + NoExclusive + } + fn subscribe_rx(&self) -> Rx { + let (tx, rx) = mpsc::channel(8); + *self.0.rx.lock() = Some(tx); + Rx(rx) + } + fn subscribe_reset(&self) -> Rx { + Rx(mpsc::channel(1).1) + } +} + +fn config(eui: u8, nwk: u16) -> RadioConfig { + RadioConfig { + channel: 15, + tx_power: 0, + short_address: Nwk(nwk), + extended_address: Eui64([eui; 8]), + pan_id: PanId(0x1234), + promiscuous: false, + rx_on_when_idle: true, + frame_pending_short: vec![], + frame_pending_extended: vec![], + } +} + +/// Data frame, short dst/src, PAN compression, ack request as given. +fn data_frame(dst: u16, src: u16, ack: bool) -> TxFrame { + let fcf: u16 = 0x0001 | (u16::from(ack) << 5) | (1 << 6) | (2 << 10) | (2 << 14); + let mut psdu = fcf.to_le_bytes().to_vec(); + psdu.push(7); + psdu.extend_from_slice(&0x1234u16.to_le_bytes()); + psdu.extend_from_slice(&dst.to_le_bytes()); + psdu.extend_from_slice(&src.to_le_bytes()); + psdu.extend_from_slice(b"hello"); + TxFrame { + psdu, + channel: None, + csma_ca: true, + max_frame_retries: 3, + max_csma_backoffs: 4, + security_processed: true, + } +} + +async fn node( + port: u16, + peer: u16, + eui: u8, + nwk: u16, + mdns: bool, +) -> (TrelPhy, Arc) { + let radio = Arc::new(FakeState::default()); + let phy = TrelPhy::new( + FakeRadio(Arc::clone(&radio)), + TrelConfig { + port, + static_peers: if mdns { + vec![] + } else { + vec![([127, 0, 0, 1], peer).into()] + }, + mdns, + deny: DenyList::default(), + }, + ) + .await + .unwrap(); + phy.reconfigure(&config(eui, nwk)).await.unwrap(); + (phy, radio) +} + +#[tokio::test(flavor = "multi_thread")] +async fn static_peers_broadcast_then_unicast() { + let (a, radio_a) = node(47101, 47102, 0xAA, 0x0000, false).await; + let (b, radio_b) = node(47102, 47101, 0xBB, 0x1111, false).await; + let mut rx_a = a.subscribe_rx(); + let mut rx_b = b.subscribe_rx(); + + // Broadcast from B: over the air and to A; A learns B + b.transmit(data_frame(0xFFFF, 0x1111, false)).await.unwrap(); + assert_eq!( + radio_b.sent.lock().len(), + 1, + "broadcast also goes over the air" + ); + let got = tokio::time::timeout(Duration::from_secs(1), rx_a.recv()) + .await + .unwrap() + .unwrap(); + assert_eq!(got.lqi, 255); + + // A unicasts to B: backbone only, acked + let result = a.transmit(data_frame(0x1111, 0x0000, true)).await.unwrap(); + assert_eq!(result, TxResult::Acked); + assert!( + radio_a.sent.lock().is_empty(), + "unicast to a peer stays off the air" + ); + let got = tokio::time::timeout(Duration::from_secs(1), rx_b.recv()) + .await + .unwrap() + .unwrap(); + assert_eq!(&got.psdu[got.psdu.len() - 5..], b"hello"); + + // Unknown destination: falls through to the radio + let result = a.transmit(data_frame(0x2222, 0x0000, true)).await.unwrap(); + assert_eq!(result, TxResult::NoAck); + assert_eq!(radio_a.sent.lock().len(), 1); +} + +#[tokio::test(flavor = "multi_thread")] +async fn mdns_discovery() { + let (a, _) = node(47201, 0, 0xA1, 0x0000, true).await; + let (b, _) = node(47202, 0, 0xB1, 0x2222, true).await; + let mut rx_b = b.subscribe_rx(); + + // Wait for A to resolve B over mDNS, then unicast by NWK address + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + loop { + if a.peers() + .iter() + .any(|p| p.contains("0x2222") || p.contains("2222")) + { + break; + } + assert!( + tokio::time::Instant::now() < deadline, + "no mDNS peer: {:?}", + a.peers() + ); + tokio::time::sleep(Duration::from_millis(100)).await; + } + let result = a.transmit(data_frame(0x2222, 0x0000, true)).await.unwrap(); + assert_eq!(result, TxResult::Acked); + tokio::time::timeout(Duration::from_secs(1), rx_b.recv()) + .await + .unwrap() + .unwrap(); +} diff --git a/crates/ziggurat-server/Cargo.toml b/crates/ziggurat-server/Cargo.toml index bcf586c..e64ed26 100644 --- a/crates/ziggurat-server/Cargo.toml +++ b/crates/ziggurat-server/Cargo.toml @@ -12,6 +12,7 @@ repository.workspace = true ziggurat-spinel.workspace = true ziggurat-phy.workspace = true ziggurat-phy-spinel.workspace = true +ziggurat-phy-trel = { path = "../ziggurat-phy-trel" } ziggurat-driver.workspace = true ziggurat-zigbee.workspace = true ziggurat-protocol.workspace = true @@ -23,7 +24,7 @@ cobs = "0.5" tracing-subscriber = { version = "0.3", features = ["env-filter", "chrono"] } tokio = { version = "1.53.1", features = ["rt-multi-thread", "macros", "time", "sync", "net", "io-util", "io-std"] } tokio-serial = "5.5" -tokio-tungstenite = { version = "0.30", default-features = false, features = ["handshake"] } +tokio-tungstenite = { version = "0.30", default-features = false, features = ["handshake", "connect"] } [[bin]] name = "ziggurat" diff --git a/crates/ziggurat-server/src/main.rs b/crates/ziggurat-server/src/main.rs index 4256a98..bc627d6 100644 --- a/crates/ziggurat-server/src/main.rs +++ b/crates/ziggurat-server/src/main.rs @@ -21,6 +21,12 @@ use ziggurat_driver::zigbee_stack::{SendHandle, Tunables, ZigbeeNotification, Zi use ziggurat_driver::ziggurat_ieee_802154::types::{Eui64, Nwk, PanId}; use ziggurat_phy::{RadioConfig, RadioPhy, Receiver}; use ziggurat_phy_spinel::SpinelPhy; +use ziggurat_phy_trel::{DenyList, TrelConfig, TrelPhy}; + +mod satellite; + +/// SPIKE: every radio is wrapped in the (optionally disabled) TREL-like backbone. +pub type Phy = TrelPhy; use ziggurat_protocol::{self as proto}; use ziggurat_spinel::client::{RcpTransport, SpinelClient}; @@ -75,8 +81,9 @@ pub struct ZigguratServer { /// opened lazily by the first command that needs it and never reopened, so stack /// replacement cannot race a straggling handle — a serial port re-open failing /// with `EBUSY`, or a second concurrent connection to the same TCP endpoint - phy: AsyncMutex>>, - stack: Mutex>>>, + phy: AsyncMutex>>, + stack: Mutex>>>, + trel: TrelConfig, started: AtomicBool, notification_tx: broadcast::Sender, notification_forwarder: Mutex>>, @@ -85,11 +92,12 @@ pub struct ZigguratServer { impl ZigguratServer { /// The RCP transport is not opened and the Zigbee stack is not created until a /// client sends a command that needs them. - pub fn new(serial: SerialConfig) -> Self { + pub fn new(serial: SerialConfig, trel: TrelConfig) -> Self { let (notification_tx, _) = broadcast::channel(NOTIFICATION_HUB_DEPTH); Self { serial, + trel, phy: AsyncMutex::new(None), stack: Mutex::new(None), started: AtomicBool::new(false), @@ -149,12 +157,12 @@ impl ZigguratServer { }); } - fn current_stack(&self) -> Option>> { + fn current_stack(&self) -> Option>> { self.stack.lock().unwrap().clone() } /// The process-lifetime radio transport, opening the RCP link on first use. - async fn phy(&self) -> std::io::Result> { + pub(crate) async fn phy(&self) -> std::io::Result> { let mut phy = self.phy.lock().await; if let Some(phy) = &*phy { @@ -162,7 +170,8 @@ impl ZigguratServer { } let transport = open_transport(&self.serial).await?; - let new_phy = Arc::new(SpinelPhy::new(Arc::new(SpinelClient::new(transport)))); + let spinel = SpinelPhy::new(Arc::new(SpinelClient::new(transport))); + let new_phy = Arc::new(TrelPhy::new(spinel, self.trel.clone()).await?); *phy = Some(new_phy.clone()); drop(phy); @@ -468,13 +477,13 @@ impl ZigguratServer { // -- guards -------------------------------------------------------------------- /// The stack, in any state after `configure`. - fn configured(&self) -> Result>, proto::Error> { + fn configured(&self) -> Result>, proto::Error> { self.current_stack() .ok_or(proto::Error::Status(proto::Status::NotConfigured)) } /// The stack, if it is in the load window (configured but not started). - fn loadable(&self) -> Result>, proto::Error> { + fn loadable(&self) -> Result>, proto::Error> { match self.current_stack() { Some(_) if self.started.load(Ordering::SeqCst) => { Err(proto::Status::AlreadyStarted.into()) @@ -485,7 +494,7 @@ impl ZigguratServer { } /// The stack, if it is running. - fn running(&self) -> Result>, proto::Error> { + fn running(&self) -> Result>, proto::Error> { match self.current_stack() { Some(stack) if self.started.load(Ordering::SeqCst) => Ok(stack), Some(_) => Err(proto::Status::NotStarted.into()), @@ -647,7 +656,7 @@ impl ZigguratServer { async fn handle_get_hw_address(&self) -> Result { let phy = self.phy().await.map_err(radio_error)?; - let ieee = phy.hw_address().await.map_err(radio_error)?; + let ieee = phy.inner().hw_address().await.map_err(radio_error)?; Ok(proto::Response::HwAddress(proto::HwAddressPayload { ieee })) } @@ -666,7 +675,7 @@ impl ZigguratServer { /// (Re)initializes the Zigbee stack, but does not start it: `load_*` then /// `start_network` follow. The stack deliberately outlives client connections; /// reconfiguring replaces it wholesale. - async fn handle_configure( + pub(crate) async fn handle_configure( &self, payload: proto::ConfigurePayload, ) -> Result { @@ -702,7 +711,7 @@ impl ZigguratServer { /// permission to send commands: the network must be fully up (RCP reset handled, /// radio programmed) before replying, or the client's first command would race /// with the boot-time reset. - async fn handle_start_network(&self) -> Result { + pub(crate) async fn handle_start_network(&self) -> Result { let stack = self.loadable()?; if let Err(e) = stack.start_network().await { @@ -1005,6 +1014,32 @@ struct Args { #[arg(long, default_value = "0.0.0.0:9999")] listen: String, + /// SPIKE: UDP port of the TREL-like IP backbone; 0 disables it + #[arg(long, default_value_t = 0)] + backbone_port: u16, + + /// SPIKE: a backbone peer reachable without mDNS (`host:port`), repeatable + #[arg(long)] + backbone_peer: Vec, + + /// SPIKE: disable mDNS discovery of backbone peers + #[arg(long)] + no_mdns: bool, + + /// SPIKE: fake "out of radio range" for a device, by EUI64 (16 hex digits) or NWK + /// address (4 hex digits), repeatable + #[arg(long)] + deny: Vec, + + /// SPIKE: run as a satellite router: fetch the network from this coordinator's + /// Ziggurat API (`ws://host:port`) and start in router role + #[arg(long)] + satellite_of: Option, + + /// SPIKE: the satellite's NWK address (hex); derived from its EUI64 by default + #[arg(long)] + satellite_nwk: Option, + /// Log level (RUST_LOG still overrides, with per-module filters) #[arg(long, default_value_t = LevelFilter::DEBUG)] log_level: LevelFilter, @@ -1036,11 +1071,78 @@ fn main() -> Result<(), Box> { .init(); } - let server = Arc::new(ZigguratServer::new(SerialConfig { - device: args.device, - baudrate: args.baudrate, - flow_control: args.flow_control, - })); + let mut deny = DenyList::default(); + for entry in &args.deny { + match entry.len() { + 16 => { + deny.eui64.insert(Eui64::try_from_hex(entry)?); + } + 4 => { + deny.nwk.insert(Nwk::try_from_hex(entry)?); + } + _ => return Err(format!("bad --deny entry {entry:?}").into()), + } + } + + let server = Arc::new(ZigguratServer::new( + SerialConfig { + device: args.device, + baudrate: args.baudrate, + flow_control: args.flow_control, + }, + TrelConfig { + port: args.backbone_port, + static_peers: args.backbone_peer, + mdns: !args.no_mdns, + deny, + }, + )); + + if args.backbone_port != 0 { + let server = server.clone(); + tokio::spawn(async move { + let mut tick = tokio::time::interval(Duration::from_secs(30)); + loop { + tick.tick().await; + let Some(stack) = server.current_stack() else { + continue; + }; + for peer in stack.radio.peers() { + tracing::info!("[diag] backbone peer {peer}"); + } + let core = stack.state.core.lock(); + for n in core.nib.neighbors.entries() { + if n.device_type == ziggurat_driver::zigbee_stack::NwkDeviceType::EndDevice + { + continue; + } + tracing::info!( + "[diag] neighbor {:?} {:?} lqa={:?} in_cost={} out_cost={}", + n.network_address, + n.extended_address, + n.lqa(), + n.incoming_link_cost(), + n.outgoing_cost + ); + } + } + }); + } + + if let Some(coordinator) = args.satellite_of { + let nwk = args + .satellite_nwk + .as_deref() + .map(Nwk::try_from_hex) + .transpose()?; + let server = server.clone(); + tokio::spawn(async move { + if let Err(e) = satellite::bootstrap(&server, &coordinator, nwk).await { + tracing::error!("Satellite bootstrap failed: {e}"); + std::process::exit(1); + } + }); + } match args.api { ApiMode::Ws => server.run(&args.listen).await?, diff --git a/crates/ziggurat-server/src/satellite.rs b/crates/ziggurat-server/src/satellite.rs new file mode 100644 index 0000000..44620f3 --- /dev/null +++ b/crates/ziggurat-server/src/satellite.rs @@ -0,0 +1,112 @@ +//! SPIKE (throwaway): bring a Ziggurat node up as a satellite router, configured +//! out-of-band from the coordinator's own Ziggurat API instead of joining over the air. + +use futures_util::{SinkExt, StreamExt}; +use tokio_tungstenite::tungstenite::Message; +use ziggurat_driver::ziggurat_ieee_802154::types::{Eui64, Nwk}; +use ziggurat_protocol::{self as proto}; + +use crate::ZigguratServer; + +const REQUEST_ID: proto::RequestId = 1; + +pub async fn bootstrap( + server: &std::sync::Arc, + coordinator: &str, + nwk: Option, +) -> Result<(), Box> { + tracing::info!("Satellite: fetching network settings from {coordinator}"); + let info = fetch_network_info(coordinator).await?; + let coord = &info.state; + + let phy = server.phy().await?; + let ieee = phy.inner().hw_address().await?; + let nwk = nwk.unwrap_or_else(|| derive_nwk(ieee)); + + tracing::info!( + "Satellite: joining PAN {:?} (ext {:?}) on channel {} as router {nwk:?} / {ieee:?}", + coord.pan_id, + coord.extended_pan_id, + coord.channel + ); + + let payload = proto::ConfigurePayload { + role: proto::NodeRole::Router, + source_routing: false, + state: proto::NetworkState { + nwk_address: nwk, + ieee_address: ieee, + // Stateless, but restarts must not reuse counters (neighbours would drop + // our frames as replays): seed from the wall clock, 10 frames/s of headroom + network_key_tx_counter: clock_counter(), + aps_frame_counter: clock_counter(), + has_tclk_seed: false, + ..coord.clone() + }, + }; + + server + .handle_configure(payload) + .await + .map_err(|_| "configure failed")?; + server + .handle_start_network() + .await + .map_err(|_| "start_network failed")?; + + tracing::info!("Satellite: router is up"); + Ok(()) +} + +async fn fetch_network_info( + url: &str, +) -> Result> { + let (mut ws, _) = tokio_tungstenite::connect_async(url).await?; + + let command = proto::RequestCommand::GetNetworkInfo as u8; + let request = proto::envelope(proto::FrameType::Request, command, REQUEST_ID); + ws.send(Message::Binary(request.into())).await?; + + while let Some(message) = ws.next().await { + let Message::Binary(data) = message? else { + continue; + }; + let Some((header, consumed)) = proto::Header::parse(&data) else { + continue; + }; + if header.frame_type != proto::FrameType::Response || header.command != command { + continue; // the hello notification, other clients' traffic + } + let body = &data[consumed..]; + if body.first() != Some(&(proto::Status::Ok as u8)) { + return Err(format!("coordinator refused get_network_info: {body:02X?}").into()); + } + let info: proto::NetworkInfoPayload = + proto::parse(&body[1..]).ok_or("malformed network info")?; + if !info.started { + return Err("coordinator network is not started".into()); + } + let _ = ws.close(None).await; + return Ok(info); + } + Err("coordinator closed the connection".into()) +} + +/// A stable address from the EUI64, outside the reserved 0x0000 and 0xFFF8.. ranges. +/// No conflict check: the stack's address conflict detection will report one. +fn derive_nwk(ieee: Eui64) -> Nwk { + let mut hash: u32 = 2_166_136_261; + for byte in ieee.0 { + hash = (hash ^ u32::from(byte)).wrapping_mul(16_777_619); + } + let nwk = ((hash ^ (hash >> 16)) as u16) % 0xFFF7 + 1; + Nwk(nwk) +} + +fn clock_counter() -> u32 { + let secs = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |d| d.as_secs()); + // Seconds since 2026-01-01, times 10: fits u32 for ~13 years + (secs.saturating_sub(1_767_225_600) * 10) as u32 +} From 7828269f5906bd84fe9f6d00a5c1a67aca80746a Mon Sep 17 00:00:00 2001 From: Igor Yamolov Date: Tue, 10 Nov 2026 13:31:16 +0100 Subject: [PATCH 02/22] SPIKE: stable peer addresses, self-configured test coordinator - Pick mDNS addresses deterministically (LAN before container bridges and loopback) and stop overriding a peer's address once it has sent us datagrams - --test-network/--test-channel bring a coordinator up on a private test network without ZHA, for bench runs Co-Authored-By: Claude Opus 5.5 --- crates/ziggurat-phy-trel/src/lib.rs | 43 +++++++++++--- crates/ziggurat-server/src/main.rs | 20 +++++++ crates/ziggurat-server/src/satellite.rs | 74 +++++++++++++++++++++++++ 3 files changed, 129 insertions(+), 8 deletions(-) diff --git a/crates/ziggurat-phy-trel/src/lib.rs b/crates/ziggurat-phy-trel/src/lib.rs index 13ac178..c38af57 100644 --- a/crates/ziggurat-phy-trel/src/lib.rs +++ b/crates/ziggurat-phy-trel/src/lib.rs @@ -316,7 +316,13 @@ impl Backbone { let nwk = info .get_property_val_str("nwk") .and_then(|s| Nwk::try_from_hex(s).ok()); - let Some(ip) = info.get_addresses_v4().into_iter().next() else { + // mDNS lists every interface (Docker bridges, loopback, ...): + // pick deterministically, LAN first + let Some(ip) = info + .get_addresses_v4() + .into_iter() + .min_by_key(|ip| (address_rank(*ip), *ip)) + else { continue; }; let addr = SocketAddr::new(IpAddr::V4(ip), info.get_port()); @@ -328,14 +334,21 @@ impl Backbone { if eui64 == identity.eui64 || pan_id != Some(identity.pan_id) { continue; } - tracing::info!("TREL: mDNS peer {eui64:?} / {nwk:?} at {addr}"); - let peer = shared.peers.entry(eui64).or_insert(Peer { - addr, - nwk, - source: PeerSource::Mdns, - last_rx: None, + let peer = shared.peers.entry(eui64).or_insert_with(|| { + tracing::info!("TREL: mDNS peer {eui64:?} / {nwk:?} at {addr}"); + Peer { + addr, + nwk, + source: PeerSource::Mdns, + last_rx: None, + } }); - peer.addr = addr; + // Once the peer has talked to us, its datagrams' source address + // is authoritative + if peer.last_rx.is_none() && peer.addr != addr { + tracing::info!("TREL: mDNS peer {eui64:?} moved to {addr}"); + peer.addr = addr; + } if nwk.is_some() { peer.nwk = nwk; } @@ -740,6 +753,20 @@ impl MacHeader { } } +/// Lower is better: 192.168/16, 10/8, other non-loopback, 172.16/12 (usually +/// container bridges), link-local, loopback. +const fn address_rank(ip: std::net::Ipv4Addr) -> u8 { + let o = ip.octets(); + match o { + [192, 168, ..] => 0, + [10, ..] => 1, + [172, 16..=31, ..] => 3, + [169, 254, ..] => 4, + [127, ..] => 5, + _ => 2, + } +} + fn hex_be(bytes: &[u8]) -> String { bytes.iter().map(|b| format!("{b:02x}")).collect() } diff --git a/crates/ziggurat-server/src/main.rs b/crates/ziggurat-server/src/main.rs index bc627d6..c9b5c48 100644 --- a/crates/ziggurat-server/src/main.rs +++ b/crates/ziggurat-server/src/main.rs @@ -1036,6 +1036,15 @@ struct Args { #[arg(long)] satellite_of: Option, + /// SPIKE: start as coordinator of a private test network kept in this file + /// (created on first use), instead of waiting for a client to configure us + #[arg(long)] + test_network: Option, + + /// SPIKE: channel for a newly created --test-network + #[arg(long, default_value_t = 25)] + test_channel: u8, + /// SPIKE: the satellite's NWK address (hex); derived from its EUI64 by default #[arg(long)] satellite_nwk: Option, @@ -1129,6 +1138,17 @@ fn main() -> Result<(), Box> { }); } + if let Some(path) = args.test_network { + let server = server.clone(); + let channel = args.test_channel; + tokio::spawn(async move { + if let Err(e) = satellite::test_coordinator(&server, &path, channel).await { + tracing::error!("Test network bring-up failed: {e}"); + std::process::exit(1); + } + }); + } + if let Some(coordinator) = args.satellite_of { let nwk = args .satellite_nwk diff --git a/crates/ziggurat-server/src/satellite.rs b/crates/ziggurat-server/src/satellite.rs index 44620f3..585f2c3 100644 --- a/crates/ziggurat-server/src/satellite.rs +++ b/crates/ziggurat-server/src/satellite.rs @@ -110,3 +110,77 @@ fn clock_counter() -> u32 { // Seconds since 2026-01-01, times 10: fits u32 for ~13 years (secs.saturating_sub(1_767_225_600) * 10) as u32 } + +/// SPIKE: bring this node up as coordinator of a private test network, created on +/// first use and kept in `path` (`key=value` lines) so restarts keep the network. +pub async fn test_coordinator( + server: &std::sync::Arc, + path: &str, + channel: u8, +) -> Result<(), Box> { + use ziggurat_driver::ziggurat_ieee_802154::types::{Key, PanId}; + + if !std::path::Path::new(path).exists() { + let mut random = [0u8; 26]; + std::io::Read::read_exact(&mut std::fs::File::open("/dev/urandom")?, &mut random)?; + let hex = |b: &[u8]| b.iter().map(|x| format!("{x:02x}")).collect::(); + let pan = u16::from_le_bytes([random[0], random[1]]) % 0xFFF0 + 1; + std::fs::write( + path, + format!( + "channel={channel}\npan={pan:04x}\nepid={}\nkey={}\n", + hex(&random[2..10]), + hex(&random[10..26]) + ), + )?; + tracing::info!("Test network: created {path}"); + } + + let text = std::fs::read_to_string(path)?; + let get = |k: &str| { + text.lines() + .find_map(|l| l.strip_prefix(&format!("{k}="))) + .ok_or_else(|| format!("{path}: missing {k}")) + }; + let mut key = [0u8; 16]; + let key_hex = get("key")?; + for (i, byte) in key.iter_mut().enumerate() { + *byte = u8::from_str_radix(&key_hex[i * 2..i * 2 + 2], 16)?; + } + + let phy = server.phy().await?; + let ieee = phy.inner().hw_address().await?; + + let payload = proto::ConfigurePayload { + role: proto::NodeRole::Coordinator, + source_routing: true, + state: proto::NetworkState { + channel: get("channel")?.parse()?, + nwk_update_id: 0, + pan_id: PanId::try_from_hex(get("pan")?)?, + extended_pan_id: Eui64::try_from_hex(get("epid")?)?, + nwk_address: Nwk(0x0000), + ieee_address: ieee, + network_key: Key(key), + network_key_seq: 0, + network_key_tx_counter: clock_counter(), + tc_link_key: Key(*b"ZigBeeAlliance09"), + has_tclk_seed: false, + tclk_seed: Key([0; 16]), + tclk_flavor: proto::TclkFlavorId::ZStack, + tx_power: 8, + aps_frame_counter: clock_counter(), + }, + }; + tracing::info!("Test network: coordinator {ieee:?} from {path}"); + server + .handle_configure(payload) + .await + .map_err(|_| "configure failed")?; + server + .handle_start_network() + .await + .map_err(|_| "start_network failed")?; + tracing::info!("Test network: coordinator is up"); + Ok(()) +} From 408d363d1a924e8caba1f5352342aaa532ca9b63 Mon Sep 17 00:00:00 2001 From: Igor Yamolov Date: Tue, 10 Nov 2026 13:42:10 +0100 Subject: [PATCH 03/22] SPIKE: ZLOE backbone interface option Name the backbone ZLOE (Zigbee Link Over Ethernet) and add --zloe-interface to confine it to one network interface: the UDP socket binds that interface's IPv4 address, mDNS runs only there and advertises only that address, and same-subnet peer addresses win. --backbone-* flags remain as aliases of the new --zloe-* names; the mDNS service type is now _zloe._udp. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 1 + crates/ziggurat-phy-trel/Cargo.toml | 1 + crates/ziggurat-phy-trel/SPIKE.md | 16 ++- crates/ziggurat-phy-trel/src/lib.rs | 122 +++++++++++++++------ crates/ziggurat-phy-trel/tests/loopback.rs | 1 + crates/ziggurat-server/src/main.rs | 32 +++--- 6 files changed, 124 insertions(+), 49 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c099752..6562621 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1967,6 +1967,7 @@ dependencies = [ name = "ziggurat-phy-trel" version = "0.1.0" dependencies = [ + "if-addrs", "mdns-sd", "parking_lot", "tokio", diff --git a/crates/ziggurat-phy-trel/Cargo.toml b/crates/ziggurat-phy-trel/Cargo.toml index 22c684d..fbcc2cd 100644 --- a/crates/ziggurat-phy-trel/Cargo.toml +++ b/crates/ziggurat-phy-trel/Cargo.toml @@ -16,6 +16,7 @@ parking_lot = "0.12.5" tokio = { version = "1.53.1", features = ["rt", "time", "sync", "net", "macros"] } tracing = "0.1" mdns-sd = "0.21.7" +if-addrs = "0.15" [dev-dependencies] tokio = { version = "1.53.1", features = ["rt-multi-thread", "macros", "time", "sync", "net"] } diff --git a/crates/ziggurat-phy-trel/SPIKE.md b/crates/ziggurat-phy-trel/SPIKE.md index 6af7797..1092037 100644 --- a/crates/ziggurat-phy-trel/SPIKE.md +++ b/crates/ziggurat-phy-trel/SPIKE.md @@ -1,4 +1,4 @@ -# SPIKE: TREL-like IP backbone for satellite routers (zigpy/ziggurat#84) +# SPIKE: ZLOE (Zigbee Link Over Ethernet), a TREL-like IP backbone, for satellite routers (zigpy/ziggurat#84) Throwaway. Answers: can Ziggurat routers, configured out-of-band and linked to the coordinator over IP, extend coverage without any NWK changes? @@ -11,7 +11,7 @@ coordinator over IP, extend coverage without any NWK changes? Broadcast → air **and** every peer (NWK broadcast dedup absorbs duplicates). - Frames from peers are reported with LQI 255 (on either link), so NWK link cost to a peer is 1 and routing prefers the backbone. -- Peers: mDNS `_zigtrel._udp` (TXT `eui64`, `nwk`, `pan`, `v`), `--backbone-peer`, +- Peers: mDNS `_zloe._udp` (TXT `eui64`, `nwk`, `pan`, `v`), `--zloe-peer`, or learned from the first datagram. Different PAN → ignored. - Satellite: `--satellite-of ws://coord:9999` fetches `get_network_info` from the coordinator's Ziggurat API and starts in router role. NWK address derived from the @@ -27,14 +27,14 @@ Coordinator (ZHA connects here as usual): ```bash cargo run --bin ziggurat -- --device /dev/serial/by-id/ --listen 0.0.0.0:9999 \ - --backbone-port 19790 + --zloe-port 19790 ``` Satellite (same box is fine, different port/radio): ```bash cargo run --bin ziggurat -- --device /dev/serial/by-id/ --listen 127.0.0.1:9998 \ - --backbone-port 19791 --satellite-of ws://127.0.0.1:9999 + --zloe-port 19791 --satellite-of ws://127.0.0.1:9999 ``` Experiment: @@ -57,3 +57,11 @@ Loopback smoke tests (no hardware): `cargo test -p ziggurat-phy-trel`. device. Fine once routes are established; misleading for direct joins. - No peer authentication (NWK frames are encrypted, but a LAN host could flood or replay). IPv4 only. No children on satellites. Satellite isn't announced to ZHA. + +## ZLOE interface + +`--zloe-interface ` confines the backbone to one interface: UDP binds that +interface's IPv4 address, mDNS runs only there and advertises only that address, and +peer addresses on the same subnet are preferred. Without it, all interfaces are used +(mDNS then advertises Docker bridges and loopback too). The old `--backbone-*` flag +names still work as aliases. diff --git a/crates/ziggurat-phy-trel/src/lib.rs b/crates/ziggurat-phy-trel/src/lib.rs index c38af57..858bc50 100644 --- a/crates/ziggurat-phy-trel/src/lib.rs +++ b/crates/ziggurat-phy-trel/src/lib.rs @@ -1,4 +1,5 @@ -//! SPIKE (throwaway): a TREL-like IP backbone, wrapped around any [`RadioPhy`]. +//! SPIKE (throwaway): ZLOE (Zigbee Link Over Ethernet), a TREL-like IP backbone, +//! wrapped around any [`RadioPhy`]. //! //! The backbone behaves as a second 802.15.4 "radio" that only reaches other Ziggurat //! nodes. Raw 802.15.4 frames (FCS stripped) are carried in UDP datagrams: @@ -12,7 +13,7 @@ //! on), so NWK link costs to backbone peers are 1 and routing prefers the backbone //! without any NWK changes. //! -//! Peers are discovered with mDNS/DNS-SD (`_zigtrel._udp`), configured statically, or +//! Peers are discovered with mDNS/DNS-SD (`_zloe._udp`), configured statically, or //! learned from the first datagram they send us. //! //! A test-only deny list fakes "out of radio range": over-the-air frames whose MAC @@ -21,7 +22,7 @@ //! devices at the hardware level; see the spike notes. use std::collections::{HashMap, HashSet, VecDeque}; -use std::net::{IpAddr, SocketAddr}; +use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use std::sync::Arc; use std::sync::atomic::{AtomicU16, Ordering}; use std::time::{Duration, Instant}; @@ -32,7 +33,7 @@ use tokio::sync::{mpsc, oneshot}; use ziggurat_ieee_802154::types::{Eui64, Nwk, PanId}; use ziggurat_phy::{RadioConfig, RadioError, RadioPhy, Receiver, RxFrame, TxFrame, TxResult}; -pub const SERVICE_TYPE: &str = "_zigtrel._udp.local."; +pub const SERVICE_TYPE: &str = "_zloe._udp.local."; const MAGIC: [u8; 2] = *b"ZT"; const VERSION: u8 = 1; const HEADER_LEN: usize = 19; @@ -70,9 +71,28 @@ pub struct TrelConfig { /// Peers reachable without discovery; their identity is learned from traffic pub static_peers: Vec, pub mdns: bool, + /// Network interface (e.g. `eth0`) the backbone is confined to: the UDP socket + /// binds its IPv4 address and mDNS only runs there. `None`: all interfaces. + pub interface: Option, pub deny: DenyList, } +/// The IPv4 address and netmask of a named interface. +fn interface_v4(name: &str) -> std::io::Result<(Ipv4Addr, Ipv4Addr)> { + if_addrs::get_if_addrs()? + .into_iter() + .find_map(|iface| match iface.addr { + if_addrs::IfAddr::V4(v4) if iface.name == name => Some((v4.ip, v4.netmask)), + _ => None, + }) + .ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::NotFound, + format!("ZLOE interface {name:?} has no IPv4 address"), + ) + }) +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum PeerSource { Mdns, @@ -114,6 +134,8 @@ struct Backbone { static_peers: Vec, mdns: Option, port: u16, + /// Set when confined to one interface: its address and netmask + local_net: Option<(Ipv4Addr, Ipv4Addr)>, } pub struct TrelPhy { @@ -140,14 +162,31 @@ impl TrelPhy

{ let backbone = if config.port == 0 { None } else { - let socket = Arc::new(UdpSocket::bind(("0.0.0.0", config.port)).await?); - tracing::info!("TREL backbone listening on UDP {}", config.port); + let local_net = config.interface.as_deref().map(interface_v4).transpose()?; + let bind_ip = local_net.map_or(Ipv4Addr::UNSPECIFIED, |(ip, _)| ip); + let socket = Arc::new(UdpSocket::bind((bind_ip, config.port)).await?); + tracing::info!( + "ZLOE backbone listening on {bind_ip}:{} ({})", + config.port, + config.interface.as_deref().unwrap_or("all interfaces") + ); let mdns = if config.mdns { match mdns_sd::ServiceDaemon::new() { - Ok(daemon) => Some(daemon), + Ok(daemon) => { + if let Some(name) = &config.interface { + let _ = daemon.disable_interface(mdns_sd::IfKind::All); + let _ = daemon.disable_interface(mdns_sd::IfKind::LoopbackV4); + if let Err(e) = + daemon.enable_interface(mdns_sd::IfKind::Name(name.clone())) + { + tracing::warn!("ZLOE: mDNS on {name} failed: {e}"); + } + } + Some(daemon) + } Err(e) => { - tracing::warn!("TREL: mDNS unavailable, static peers only: {e}"); + tracing::warn!("ZLOE: mDNS unavailable, static peers only: {e}"); None } } @@ -163,6 +202,7 @@ impl TrelPhy

{ static_peers: config.static_peers, mdns, port: config.port, + local_net, }; backbone.spawn_udp_reader(); backbone.spawn_mdns_browser(); @@ -194,13 +234,13 @@ impl Backbone { let (len, from) = match socket.recv_from(&mut buf).await { Ok(r) => r, Err(e) => { - tracing::warn!("TREL: UDP receive failed: {e}"); + tracing::warn!("ZLOE: UDP receive failed: {e}"); tokio::time::sleep(Duration::from_millis(100)).await; continue; } }; let Some(header) = WireHeader::parse(&buf[..len]) else { - tracing::debug!("TREL: dropping malformed datagram from {from}"); + tracing::debug!("ZLOE: dropping malformed datagram from {from}"); continue; }; @@ -224,7 +264,7 @@ impl Backbone { // Learn (or refresh) the peer from its own traffic let peer = shared.peers.entry(header.eui64).or_insert_with(|| { tracing::info!( - "TREL: learned peer {:?} / {:?} at {from}", + "ZLOE: learned peer {:?} / {:?} at {from}", header.eui64, header.nwk ); @@ -279,7 +319,7 @@ impl Backbone { if let Some((Some(tx), frame)) = deliver { tracing::trace!( - "TREL: rx {} bytes from {:?}", + "ZLOE: rx {} bytes from {:?}", frame.psdu.len(), header.eui64 ); @@ -294,11 +334,12 @@ impl Backbone { let events = match daemon.browse(SERVICE_TYPE) { Ok(events) => events, Err(e) => { - tracing::warn!("TREL: mDNS browse failed: {e}"); + tracing::warn!("ZLOE: mDNS browse failed: {e}"); return; } }; let shared = Arc::clone(&self.shared); + let local_net = self.local_net; tokio::spawn(async move { while let Ok(event) = events.recv_async().await { @@ -318,11 +359,13 @@ impl Backbone { .and_then(|s| Nwk::try_from_hex(s).ok()); // mDNS lists every interface (Docker bridges, loopback, ...): // pick deterministically, LAN first - let Some(ip) = info - .get_addresses_v4() - .into_iter() - .min_by_key(|ip| (address_rank(*ip), *ip)) - else { + let Some(ip) = info.get_addresses_v4().into_iter().min_by_key(|ip| { + let on_our_net = local_net.is_some_and(|(ours, mask)| { + u32::from(*ip) & u32::from(mask) + == u32::from(ours) & u32::from(mask) + }); + (!on_our_net, address_rank(*ip), *ip) + }) else { continue; }; let addr = SocketAddr::new(IpAddr::V4(ip), info.get_port()); @@ -335,7 +378,7 @@ impl Backbone { continue; } let peer = shared.peers.entry(eui64).or_insert_with(|| { - tracing::info!("TREL: mDNS peer {eui64:?} / {nwk:?} at {addr}"); + tracing::info!("ZLOE: mDNS peer {eui64:?} / {nwk:?} at {addr}"); Peer { addr, nwk, @@ -346,7 +389,7 @@ impl Backbone { // Once the peer has talked to us, its datagrams' source address // is authoritative if peer.last_rx.is_none() && peer.addr != addr { - tracing::info!("TREL: mDNS peer {eui64:?} moved to {addr}"); + tracing::info!("ZLOE: mDNS peer {eui64:?} moved to {addr}"); peer.addr = addr; } if nwk.is_some() { @@ -367,7 +410,7 @@ impl Backbone { .get(&eui64) .is_some_and(|p| p.source == PeerSource::Mdns) { - tracing::info!("TREL: mDNS peer {eui64:?} went away"); + tracing::info!("ZLOE: mDNS peer {eui64:?} went away"); shared.peers.remove(&eui64); } } @@ -395,19 +438,34 @@ impl Backbone { .collect(); let host = format!("ziggurat-{instance}.local."); - match mdns_sd::ServiceInfo::new(SERVICE_TYPE, &instance, &host, "", self.port, properties) { + let ip = self + .local_net + .map(|(ip, _)| ip.to_string()) + .unwrap_or_default(); + match mdns_sd::ServiceInfo::new( + SERVICE_TYPE, + &instance, + &host, + ip.as_str(), + self.port, + properties, + ) { Ok(info) => { - let info = info.enable_addr_auto(); + let info = if self.local_net.is_some() { + info + } else { + info.enable_addr_auto() + }; let fullname = info.get_fullname().to_string(); match daemon.register(info) { Ok(()) => { - tracing::info!("TREL: advertising {fullname}"); + tracing::info!("ZLOE: advertising {fullname}"); shared.mdns_fullname = Some(fullname); } - Err(e) => tracing::warn!("TREL: mDNS register failed: {e}"), + Err(e) => tracing::warn!("ZLOE: mDNS register failed: {e}"), } } - Err(e) => tracing::warn!("TREL: bad mDNS service info: {e}"), + Err(e) => tracing::warn!("ZLOE: bad mDNS service info: {e}"), } } @@ -483,13 +541,13 @@ impl Backbone { } if tokio::time::timeout(ACK_TIMEOUT, rx).await.is_ok() { if attempt > 0 { - tracing::debug!("TREL: unicast to {to} acked after {} retries", attempt); + tracing::debug!("ZLOE: unicast to {to} acked after {} retries", attempt); } return true; } } self.pending.lock().remove(&seq); - tracing::debug!("TREL: unicast to {to} not acked"); + tracing::debug!("ZLOE: unicast to {to} not acked"); false } } @@ -553,18 +611,18 @@ impl RadioPhy for TrelPhy

{ .send_unicast(peer, &frame.psdu, mac.ack_request) .await { - tracing::trace!("TREL: tx to {:?} via backbone", mac.dest); + tracing::trace!("ZLOE: tx to {:?} via backbone", mac.dest); return Ok(TxResult::Acked); } tracing::info!( - "TREL: backbone to {:?} failed, falling back to air", + "ZLOE: backbone to {:?} failed, falling back to air", mac.dest ); } } if self.deny.denies(&mac.dest) { - tracing::trace!("TREL deny: not transmitting to {:?}", mac.dest); + tracing::trace!("ZLOE deny: not transmitting to {:?}", mac.dest); return Ok(if mac.ack_request { TxResult::NoAck } else { @@ -597,7 +655,7 @@ impl RadioPhy for TrelPhy

{ while let Some(mut frame) = inner_rx.recv().await { if let Some(mac) = MacHeader::parse(&frame.psdu) { if deny.denies(&mac.source) { - tracing::trace!("TREL deny: dropping air frame from {:?}", mac.source); + tracing::trace!("ZLOE deny: dropping air frame from {:?}", mac.source); continue; } if has_backbone { diff --git a/crates/ziggurat-phy-trel/tests/loopback.rs b/crates/ziggurat-phy-trel/tests/loopback.rs index 7ec9b8b..7a681ef 100644 --- a/crates/ziggurat-phy-trel/tests/loopback.rs +++ b/crates/ziggurat-phy-trel/tests/loopback.rs @@ -117,6 +117,7 @@ async fn node( vec![([127, 0, 0, 1], peer).into()] }, mdns, + interface: None, deny: DenyList::default(), }, ) diff --git a/crates/ziggurat-server/src/main.rs b/crates/ziggurat-server/src/main.rs index c9b5c48..0361726 100644 --- a/crates/ziggurat-server/src/main.rs +++ b/crates/ziggurat-server/src/main.rs @@ -1014,17 +1014,22 @@ struct Args { #[arg(long, default_value = "0.0.0.0:9999")] listen: String, - /// SPIKE: UDP port of the TREL-like IP backbone; 0 disables it - #[arg(long, default_value_t = 0)] - backbone_port: u16, + /// SPIKE: UDP port of the ZLOE (Zigbee Link Over Ethernet) backbone; 0 disables it + #[arg(long, alias = "backbone-port", default_value_t = 0)] + zloe_port: u16, - /// SPIKE: a backbone peer reachable without mDNS (`host:port`), repeatable - #[arg(long)] - backbone_peer: Vec, + /// SPIKE: network interface the ZLOE backbone is confined to (e.g. `eth0`): UDP + /// binds its IPv4 address and mDNS runs only there. Default: all interfaces + #[arg(long, alias = "backbone-interface")] + zloe_interface: Option, - /// SPIKE: disable mDNS discovery of backbone peers - #[arg(long)] - no_mdns: bool, + /// SPIKE: a ZLOE peer reachable without mDNS (`host:port`), repeatable + #[arg(long, alias = "backbone-peer")] + zloe_peer: Vec, + + /// SPIKE: disable mDNS discovery of ZLOE peers + #[arg(long, alias = "no-mdns")] + zloe_no_mdns: bool, /// SPIKE: fake "out of radio range" for a device, by EUI64 (16 hex digits) or NWK /// address (4 hex digits), repeatable @@ -1100,14 +1105,15 @@ fn main() -> Result<(), Box> { flow_control: args.flow_control, }, TrelConfig { - port: args.backbone_port, - static_peers: args.backbone_peer, - mdns: !args.no_mdns, + port: args.zloe_port, + static_peers: args.zloe_peer, + mdns: !args.zloe_no_mdns, + interface: args.zloe_interface, deny, }, )); - if args.backbone_port != 0 { + if args.zloe_port != 0 { let server = server.clone(); tokio::spawn(async move { let mut tick = tokio::time::interval(Duration::from_secs(30)); From 4ce70abf082555053449b843252837efa08df403 Mon Sep 17 00:00:00 2001 From: Igor Yamolov Date: Tue, 10 Nov 2026 14:27:44 +0100 Subject: [PATCH 04/22] SPIKE: bench tooling for third-party routers on the test network - zloe-ctl: open joins and send On/Off through a node's API, no ZHA - --test-network answers Node_Desc_req for the coordinator itself (zigpy's job normally); ZBOSS joiners leave without it - Deliver to router children directly, the zigpy/ziggurat#88 workaround (fix proper is #89), so ESP32-C6 ZBOSS routers stay joined Co-Authored-By: Claude Opus 5.5 --- crates/ziggurat-server/Cargo.toml | 5 + crates/ziggurat-server/src/bin/zloe_ctl.rs | 124 +++++++++++++++++++++ crates/ziggurat-server/src/satellite.rs | 70 ++++++++++++ crates/ziggurat-zigbee/src/nwk/next_hop.rs | 12 ++ 4 files changed, 211 insertions(+) create mode 100644 crates/ziggurat-server/src/bin/zloe_ctl.rs diff --git a/crates/ziggurat-server/Cargo.toml b/crates/ziggurat-server/Cargo.toml index e64ed26..ae1b39d 100644 --- a/crates/ziggurat-server/Cargo.toml +++ b/crates/ziggurat-server/Cargo.toml @@ -30,3 +30,8 @@ tokio-tungstenite = { version = "0.30", default-features = false, features = ["h name = "ziggurat" path = "src/main.rs" doc = false + +[[bin]] +name = "zloe-ctl" +path = "src/bin/zloe_ctl.rs" +doc = false diff --git a/crates/ziggurat-server/src/bin/zloe_ctl.rs b/crates/ziggurat-server/src/bin/zloe_ctl.rs new file mode 100644 index 0000000..aa309b7 --- /dev/null +++ b/crates/ziggurat-server/src/bin/zloe_ctl.rs @@ -0,0 +1,124 @@ +//! SPIKE (throwaway): poke a Ziggurat node's API for the ZLOE bench without ZHA. +//! +//! zloe-ctl permit +//! zloe-ctl onoff +//! zloe-ctl listen (print notifications) + +use futures_util::{SinkExt, StreamExt}; +use tokio_tungstenite::tungstenite::Message; +use ziggurat_driver::ziggurat_ieee_802154::types::{Eui64, Nwk}; +use ziggurat_protocol::{self as proto}; + +type Error = Box; + +#[tokio::main(flavor = "current_thread")] +async fn main() -> Result<(), Error> { + let args: Vec = std::env::args().skip(1).collect(); + let [url, command, rest @ ..] = &args[..] else { + return Err( + "usage: zloe-ctl permit | onoff | listen ".into(), + ); + }; + let (mut ws, _) = tokio_tungstenite::connect_async(url.as_str()).await?; + + let (opcode, payload) = match (command.as_str(), rest) { + ("permit", [secs]) => { + let mut body = Vec::new(); + proto::append( + &mut body, + &proto::PermitJoinsPayload { + duration: secs.parse()?, + accept_direct_joins: true, + }, + ); + (proto::RequestCommand::PermitJoins, body) + } + ("onoff", [nwk, action]) => { + let zcl_command = match action.as_str() { + "off" => 0x00, + "on" => 0x01, + "toggle" => 0x02, + _ => return Err("action must be on, off or toggle".into()), + }; + // ZCL: cluster-specific, client->server, default response enabled + let seq = (std::process::id() & 0xFF) as u8; + let mut body = Vec::new(); + proto::append( + &mut body, + &proto::SendUnicastPayload { + flags: proto::SendUnicastFlags { + has_eui64: false, + aps_ack: true, + aps_encryption: false, + sleepy_destination: false, + force_aps_seq: false, + }, + destination: Nwk::try_from_hex(nwk)?, + destination_eui64: Eui64([0; 8]), + profile_id: 0x0104, + cluster_id: 0x0006, + src_ep: 1, + dst_ep: 1, + aps_seq: 0, + radius: 0, + priority: 0, + route: proto::RouteControl::StackDecides, + next_hop: None, + relays: None, + asdu: vec![0x01, seq, zcl_command], + }, + ); + (proto::RequestCommand::SendUnicast, body) + } + ("listen", [secs]) => { + let deadline = std::time::Duration::from_secs(secs.parse()?); + let _ = tokio::time::timeout(deadline, async { + while let Some(Ok(Message::Binary(data))) = ws.next().await { + if let Some((header, _)) = proto::Header::parse(&data) { + println!( + "{:?} cmd={:#04x} {:02x?}", + header.frame_type, + header.command, + &data[3..data.len().min(40)] + ); + } + } + }) + .await; + return Ok(()); + } + _ => return Err(format!("unknown command {command} {rest:?}").into()), + }; + + let mut frame = proto::envelope(proto::FrameType::Request, opcode as u8, 1); + frame.extend_from_slice(&payload); + ws.send(Message::Binary(frame.into())).await?; + + // Print everything until the response (and, for sends, the send status) arrives + let deadline = + std::time::Duration::from_secs(if opcode == proto::RequestCommand::SendUnicast { + 15 + } else { + 5 + }); + let _ = tokio::time::timeout(deadline, async { + while let Some(Ok(Message::Binary(data))) = ws.next().await { + let Some((header, consumed)) = proto::Header::parse(&data) else { + continue; + }; + let body = &data[consumed..]; + println!( + "{:?} cmd={:#04x} {:02x?}", + header.frame_type, header.command, body + ); + let done = header.frame_type == proto::FrameType::Response + && header.command == opcode as u8 + && opcode != proto::RequestCommand::SendUnicast; + if done { + break; + } + } + }) + .await; + Ok(()) +} diff --git a/crates/ziggurat-server/src/satellite.rs b/crates/ziggurat-server/src/satellite.rs index 585f2c3..6c942c6 100644 --- a/crates/ziggurat-server/src/satellite.rs +++ b/crates/ziggurat-server/src/satellite.rs @@ -182,5 +182,75 @@ pub async fn test_coordinator( .await .map_err(|_| "start_network failed")?; tracing::info!("Test network: coordinator is up"); + spawn_zdp_responder(server); Ok(()) } + +/// SPIKE: zigpy normally answers ZDP requests addressed to the coordinator. Without a +/// client, joiners (ZBOSS) give up when Node_Desc_req for 0x0000 goes unanswered (they +/// need the TC's stack revision), so answer that one ourselves. +fn spawn_zdp_responder(server: &std::sync::Arc) { + use ziggurat_driver::zigbee_stack::{ApsAck, RouteDirective, TxPriority, ZigbeeNotification}; + + let mut notifications = server.notification_tx.subscribe(); + let server = server.clone(); + tokio::spawn(async move { + loop { + let notification = match notifications.recv().await { + Ok(n) => n, + Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => continue, + Err(_) => return, + }; + let ZigbeeNotification::ReceivedApsCommand { + source, + profile_id: 0x0000, + cluster_id: 0x0002, + dst_ep: 0, + data, + .. + } = notification + else { + continue; + }; + let [seq, lo, hi, ..] = data[..] else { + continue; + }; + if u16::from_le_bytes([lo, hi]) != 0x0000 { + continue; + } + let Some(stack) = server.current_stack() else { + continue; + }; + + // Node descriptor: coordinator, 2.4 GHz, MAC caps 0x8F, manufacturer 0x0000, + // buffer 82, transfer sizes 128, server mask = primary TC + network manager, + // stack compliance revision 22 + let server_mask: u16 = 0x0001 | 0x0040 | (22 << 9); + let mut rsp = vec![seq, 0x00, 0x00, 0x00, 0x00, 0x40, 0x8F, 0x00, 0x00, 0x52]; + rsp.extend_from_slice(&128u16.to_le_bytes()); + rsp.extend_from_slice(&server_mask.to_le_bytes()); + rsp.extend_from_slice(&128u16.to_le_bytes()); + rsp.push(0x00); + + let result = stack.send_aps_unicast( + source, + 0x0000, + 0x8002, + 0, + 0, + ApsAck::None, + None, + 0, + rsp, + None, + false, + TxPriority::StackCritical, + RouteDirective::StackDecides, + ); + tracing::info!( + "Test network: answered Node_Desc_req from {source:?}: {:?}", + result.map(|_| ()) + ); + } + }); +} diff --git a/crates/ziggurat-zigbee/src/nwk/next_hop.rs b/crates/ziggurat-zigbee/src/nwk/next_hop.rs index a98c251..e1afcad 100644 --- a/crates/ziggurat-zigbee/src/nwk/next_hop.rs +++ b/crates/ziggurat-zigbee/src/nwk/next_hop.rs @@ -109,6 +109,18 @@ pub fn resolve( return NextHop::Resolved(destination); } + // SPIKE (#84 bench, workaround for zigpy/ziggurat#88; real fix in #89): a router + // that just associated with us is one hop away, but + // has no route and no outgoing cost until its first Link Status (~15 s). ZBOSS + // routers unicast Node_Desc_req to the TC right after joining and leave when the + // reply (and APS ack) never arrives, so deliver to router children directly too. + if neighbors + .entries() + .any(|entry| entry.network_address == destination && entry.is_child()) + { + return NextHop::Resolved(destination); + } + // A stored source route (concentrator behavior) wins over the routing table. if policy.is_concentrator && !policy.force_route_discovery { match routing.route_to(destination, policy.max_source_route) { From 360d7be44a9eec23b82389841f0c0e375dd8622b Mon Sep 17 00:00:00 2001 From: Igor Yamolov Date: Tue, 10 Nov 2026 14:31:56 +0100 Subject: [PATCH 05/22] SPIKE: satellites find the coordinator on the trusted backbone Treat the ZLOE backbone as trusted, like TREL: coordinators advertise role=coordinator over mDNS, and a satellite started with --satellite asks one for the network settings over ZLOE (NETINFO_REQ/RSP carrying the NetworkInfoPayload, network key included) instead of needing the coordinator's WebSocket URL. --satellite-pan picks one network when several coordinators share the LAN; --satellite-of stays as a fallback. Co-Authored-By: Claude Opus 5.5 --- crates/ziggurat-phy-trel/SPIKE.md | 12 ++- crates/ziggurat-phy-trel/src/lib.rs | 124 ++++++++++++++++++++++++ crates/ziggurat-server/src/main.rs | 42 ++++++++ crates/ziggurat-server/src/satellite.rs | 24 +++++ 4 files changed, 199 insertions(+), 3 deletions(-) diff --git a/crates/ziggurat-phy-trel/SPIKE.md b/crates/ziggurat-phy-trel/SPIKE.md index 1092037..655b2cc 100644 --- a/crates/ziggurat-phy-trel/SPIKE.md +++ b/crates/ziggurat-phy-trel/SPIKE.md @@ -13,8 +13,14 @@ coordinator over IP, extend coverage without any NWK changes? peer is 1 and routing prefers the backbone. - Peers: mDNS `_zloe._udp` (TXT `eui64`, `nwk`, `pan`, `v`), `--zloe-peer`, or learned from the first datagram. Different PAN → ignored. -- Satellite: `--satellite-of ws://coord:9999` fetches `get_network_info` from the - coordinator's Ziggurat API and starts in router role. NWK address derived from the +- Satellite: `--satellite` finds the coordinator on the backbone like TREL peers find + each other (mDNS TXT `role=coordinator`), fetches its settings with a + `NETINFO_REQ`/`NETINFO_RSP` exchange over ZLOE and starts in router role. The + backbone is treated as **trusted**: the settings, network key included, go + unauthenticated to whoever asks. `--satellite-pan ` picks a network when several + coordinators are on the LAN; the satellite may start before the coordinator (it + waits up to 60 s). `--satellite-of ws://coord:9999` (fetch via the coordinator's + Ziggurat API) remains as a fallback. NWK address derived from the EUI64 (or `--satellite-nwk`). Frame counters seeded from the clock. - `--deny ` fakes "out of radio range" (drops air frames from it, refuses air unicasts to it). @@ -34,7 +40,7 @@ Satellite (same box is fine, different port/radio): ```bash cargo run --bin ziggurat -- --device /dev/serial/by-id/ --listen 127.0.0.1:9998 \ - --zloe-port 19791 --satellite-of ws://127.0.0.1:9999 + --zloe-port 19791 --satellite ``` Experiment: diff --git a/crates/ziggurat-phy-trel/src/lib.rs b/crates/ziggurat-phy-trel/src/lib.rs index 858bc50..af3e812 100644 --- a/crates/ziggurat-phy-trel/src/lib.rs +++ b/crates/ziggurat-phy-trel/src/lib.rs @@ -39,6 +39,13 @@ const VERSION: u8 = 1; const HEADER_LEN: usize = 19; const KIND_DATA: u8 = 0; const KIND_ACK: u8 = 1; +/// Trusted-backbone bootstrap: ask a coordinator for the network settings, and its +/// answer (an encoded `NetworkInfoPayload`, opaque to this crate). +const KIND_NETINFO_REQ: u8 = 2; +const KIND_NETINFO_RSP: u8 = 3; + +/// Produces the encoded network settings a coordinator hands to satellites. +pub type NetInfoProvider = Arc Option> + Send + Sync>; const FLAG_ACK_REQUEST: u8 = 0x01; const ACK_TIMEOUT: Duration = Duration::from_millis(50); @@ -124,6 +131,11 @@ struct Shared { seen: HashMap>, rx_tx: Option>, mdns_fullname: Option, + /// Coordinators seen on the backbone (mDNS `role=coordinator`), kept even before + /// we have an identity of our own: satellites bootstrap from them + coordinators: HashMap)>, + netinfo_provider: Option, + netinfo_waiter: Option>>, } struct Backbone { @@ -220,6 +232,75 @@ impl TrelPhy

{ pub const fn inner(&self) -> &P { &self.inner } + + /// Coordinator side: hand these settings to satellites that ask over the backbone. + pub fn set_netinfo_provider(&self, provider: NetInfoProvider) { + self.shared.lock().netinfo_provider = Some(provider); + } + + /// Satellite side: find a coordinator on the backbone (on `pan`, when given) and + /// fetch its network settings. Retries until `timeout`. + pub async fn fetch_network( + &self, + pan: Option, + timeout: Duration, + ) -> std::io::Result> { + let backbone = self + .backbone + .as_ref() + .ok_or_else(|| std::io::Error::other("ZLOE backbone is disabled (--zloe-port 0)"))?; + let deadline = tokio::time::Instant::now() + timeout; + + while tokio::time::Instant::now() < deadline { + let candidates: Vec<(Eui64, SocketAddr, Option)> = { + let shared = self.shared.lock(); + shared + .coordinators + .iter() + .filter(|(_, (_, p))| pan.is_none() || *p == pan) + .map(|(eui64, (addr, p))| (*eui64, *addr, *p)) + .collect() + }; + let Some(&(eui64, addr, coordinator_pan)) = candidates.first() else { + tokio::time::sleep(Duration::from_millis(250)).await; + continue; + }; + if candidates.len() > 1 { + tracing::warn!( + "ZLOE: {} coordinators match, using {eui64:?} (PAN {coordinator_pan:?}); \ + pick one with --satellite-pan", + candidates.len() + ); + } + + let (tx, rx) = oneshot::channel(); + self.shared.lock().netinfo_waiter = Some(tx); + let seq = backbone.seq.fetch_add(1, Ordering::Relaxed); + let request = netinfo_header(KIND_NETINFO_REQ, seq, None); + tracing::info!("ZLOE: asking coordinator {eui64:?} at {addr} for network settings"); + backbone.socket.send_to(&request, addr).await?; + if let Ok(Ok(info)) = tokio::time::timeout(Duration::from_secs(1), rx).await { + return Ok(info); + } + } + Err(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "no coordinator answered on the ZLOE backbone", + )) + } +} + +/// Header for the bootstrap messages, sent before we may have an identity. +fn netinfo_header(kind: u8, seq: u16, identity: Option) -> Vec { + WireHeader { + kind, + flags: 0, + seq, + eui64: identity.map_or(Eui64([0; 8]), |i| i.eui64), + nwk: identity.map_or(Nwk(0xFFFF), |i| i.nwk), + pan_id: identity.map_or(PanId(0xFFFF), |i| i.pan_id), + } + .to_bytes() } impl Backbone { @@ -251,6 +332,32 @@ impl Backbone { continue; } + // Bootstrap exchange: the backbone is trusted, so the settings (network + // key included) go to whoever asks + if header.kind == KIND_NETINFO_REQ { + let (provider, identity) = { + let shared = shared.lock(); + (shared.netinfo_provider.clone(), shared.identity) + }; + let Some(info) = provider.and_then(|provide| provide()) else { + continue; + }; + tracing::info!( + "ZLOE: sending network settings to {from} ({:?})", + header.eui64 + ); + let mut rsp = netinfo_header(KIND_NETINFO_RSP, header.seq, identity); + rsp.extend_from_slice(&info); + let _ = socket.send_to(&rsp, from).await; + continue; + } + if header.kind == KIND_NETINFO_RSP { + if let Some(waiter) = shared.lock().netinfo_waiter.take() { + let _ = waiter.send(buf[HEADER_LEN..len].to_vec()); + } + continue; + } + let psdu = buf[HEADER_LEN..len].to_vec(); let (deliver, ack_to) = { let mut shared = shared.lock(); @@ -371,6 +478,14 @@ impl Backbone { let addr = SocketAddr::new(IpAddr::V4(ip), info.get_port()); let mut shared = shared.lock(); + if info.get_property_val_str("role") == Some("coordinator") { + let known = shared.coordinators.insert(eui64, (addr, pan_id)); + if known.is_none() { + tracing::info!( + "ZLOE: coordinator {eui64:?} (PAN {pan_id:?}) at {addr}" + ); + } + } let Some(identity) = shared.identity else { continue; }; @@ -433,6 +548,15 @@ impl Backbone { ("eui64".to_string(), instance.clone()), ("nwk".to_string(), format!("{:04x}", identity.nwk.as_u16())), ("pan".to_string(), format!("{:04x}", identity.pan_id.0)), + ( + "role".to_string(), + if identity.nwk == Nwk(0x0000) { + "coordinator" + } else { + "router" + } + .to_string(), + ), ] .into_iter() .collect(); diff --git a/crates/ziggurat-server/src/main.rs b/crates/ziggurat-server/src/main.rs index 0361726..15eaf0b 100644 --- a/crates/ziggurat-server/src/main.rs +++ b/crates/ziggurat-server/src/main.rs @@ -723,6 +723,19 @@ impl ZigguratServer { arc_self.run().await; }); + // SPIKE: on the trusted ZLOE backbone a coordinator hands its settings to any + // satellite that asks + let weak = Arc::downgrade(&stack); + stack.radio.set_netinfo_provider(Arc::new(move || { + let stack = weak.upgrade()?; + if stack.state.role != ziggurat_driver::zigbee_stack::NwkDeviceType::Coordinator { + return None; + } + let mut bytes = Vec::new(); + proto::append(&mut bytes, &proto::network_info_payload(&stack, true)); + Some(bytes) + })); + // Drain the stack's notification outbox into the server-level hub. The task is // aborted when the stack is replaced, so it doesn't need to observe a closed // channel to stop. @@ -1050,6 +1063,15 @@ struct Args { #[arg(long, default_value_t = 25)] test_channel: u8, + /// SPIKE: run as a satellite router: find the coordinator on the ZLOE backbone, + /// fetch the network settings from it and start in router role + #[arg(long)] + satellite: bool, + + /// SPIKE: with --satellite, only use a coordinator on this PAN (hex) + #[arg(long)] + satellite_pan: Option, + /// SPIKE: the satellite's NWK address (hex); derived from its EUI64 by default #[arg(long)] satellite_nwk: Option, @@ -1155,6 +1177,26 @@ fn main() -> Result<(), Box> { }); } + if args.satellite && args.satellite_of.is_none() { + let nwk = args + .satellite_nwk + .as_deref() + .map(Nwk::try_from_hex) + .transpose()?; + let pan = args + .satellite_pan + .as_deref() + .map(PanId::try_from_hex) + .transpose()?; + let server = server.clone(); + tokio::spawn(async move { + if let Err(e) = satellite::bootstrap_zloe(&server, pan, nwk).await { + tracing::error!("Satellite bootstrap failed: {e}"); + std::process::exit(1); + } + }); + } + if let Some(coordinator) = args.satellite_of { let nwk = args .satellite_nwk diff --git a/crates/ziggurat-server/src/satellite.rs b/crates/ziggurat-server/src/satellite.rs index 6c942c6..c197f9c 100644 --- a/crates/ziggurat-server/src/satellite.rs +++ b/crates/ziggurat-server/src/satellite.rs @@ -17,6 +17,30 @@ pub async fn bootstrap( ) -> Result<(), Box> { tracing::info!("Satellite: fetching network settings from {coordinator}"); let info = fetch_network_info(coordinator).await?; + start_router(server, &info, nwk).await +} + +/// Find the coordinator on the (trusted) ZLOE backbone and fetch the settings from it. +pub async fn bootstrap_zloe( + server: &std::sync::Arc, + pan: Option, + nwk: Option, +) -> Result<(), Box> { + tracing::info!("Satellite: looking for a coordinator on the ZLOE backbone"); + let phy = server.phy().await?; + let bytes = phy + .fetch_network(pan, std::time::Duration::from_secs(60)) + .await?; + let info: proto::NetworkInfoPayload = + proto::parse(&bytes).ok_or("malformed network settings from coordinator")?; + start_router(server, &info, nwk).await +} + +async fn start_router( + server: &std::sync::Arc, + info: &proto::NetworkInfoPayload, + nwk: Option, +) -> Result<(), Box> { let coord = &info.state; let phy = server.phy().await?; From 157e4fea16ec59ca4ece47eeef15fdf49720b1d5 Mon Sep 17 00:00:00 2001 From: Igor Yamolov Date: Tue, 10 Nov 2026 14:35:26 +0100 Subject: [PATCH 06/22] SPIKE: rename satellite options to --zloe-satellite-* --satellite, --satellite-pan, --satellite-nwk and --satellite-of are now --zloe-satellite, --zloe-satellite-pan, --zloe-satellite-nwk and --zloe-satellite-of. Co-Authored-By: Claude Opus 5.5 --- crates/ziggurat-phy-trel/SPIKE.md | 10 +++++----- crates/ziggurat-server/src/main.rs | 20 ++++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/crates/ziggurat-phy-trel/SPIKE.md b/crates/ziggurat-phy-trel/SPIKE.md index 655b2cc..5ab04a0 100644 --- a/crates/ziggurat-phy-trel/SPIKE.md +++ b/crates/ziggurat-phy-trel/SPIKE.md @@ -13,15 +13,15 @@ coordinator over IP, extend coverage without any NWK changes? peer is 1 and routing prefers the backbone. - Peers: mDNS `_zloe._udp` (TXT `eui64`, `nwk`, `pan`, `v`), `--zloe-peer`, or learned from the first datagram. Different PAN → ignored. -- Satellite: `--satellite` finds the coordinator on the backbone like TREL peers find +- Satellite: `--zloe-satellite` finds the coordinator on the backbone like TREL peers find each other (mDNS TXT `role=coordinator`), fetches its settings with a `NETINFO_REQ`/`NETINFO_RSP` exchange over ZLOE and starts in router role. The backbone is treated as **trusted**: the settings, network key included, go - unauthenticated to whoever asks. `--satellite-pan ` picks a network when several + unauthenticated to whoever asks. `--zloe-satellite-pan ` picks a network when several coordinators are on the LAN; the satellite may start before the coordinator (it - waits up to 60 s). `--satellite-of ws://coord:9999` (fetch via the coordinator's + waits up to 60 s). `--zloe-satellite-of ws://coord:9999` (fetch via the coordinator's Ziggurat API) remains as a fallback. NWK address derived from the - EUI64 (or `--satellite-nwk`). Frame counters seeded from the clock. + EUI64 (or `--zloe-satellite-nwk`). Frame counters seeded from the clock. - `--deny ` fakes "out of radio range" (drops air frames from it, refuses air unicasts to it). - Every 30 s with the backbone on: `[diag]` log lines with peers and router neighbours @@ -40,7 +40,7 @@ Satellite (same box is fine, different port/radio): ```bash cargo run --bin ziggurat -- --device /dev/serial/by-id/ --listen 127.0.0.1:9998 \ - --zloe-port 19791 --satellite + --zloe-port 19791 --zloe-satellite ``` Experiment: diff --git a/crates/ziggurat-server/src/main.rs b/crates/ziggurat-server/src/main.rs index 15eaf0b..e3db058 100644 --- a/crates/ziggurat-server/src/main.rs +++ b/crates/ziggurat-server/src/main.rs @@ -1052,7 +1052,7 @@ struct Args { /// SPIKE: run as a satellite router: fetch the network from this coordinator's /// Ziggurat API (`ws://host:port`) and start in router role #[arg(long)] - satellite_of: Option, + zloe_satellite_of: Option, /// SPIKE: start as coordinator of a private test network kept in this file /// (created on first use), instead of waiting for a client to configure us @@ -1066,15 +1066,15 @@ struct Args { /// SPIKE: run as a satellite router: find the coordinator on the ZLOE backbone, /// fetch the network settings from it and start in router role #[arg(long)] - satellite: bool, + zloe_satellite: bool, - /// SPIKE: with --satellite, only use a coordinator on this PAN (hex) + /// SPIKE: with --zloe-satellite, only use a coordinator on this PAN (hex) #[arg(long)] - satellite_pan: Option, + zloe_satellite_pan: Option, /// SPIKE: the satellite's NWK address (hex); derived from its EUI64 by default #[arg(long)] - satellite_nwk: Option, + zloe_satellite_nwk: Option, /// Log level (RUST_LOG still overrides, with per-module filters) #[arg(long, default_value_t = LevelFilter::DEBUG)] @@ -1177,14 +1177,14 @@ fn main() -> Result<(), Box> { }); } - if args.satellite && args.satellite_of.is_none() { + if args.zloe_satellite && args.zloe_satellite_of.is_none() { let nwk = args - .satellite_nwk + .zloe_satellite_nwk .as_deref() .map(Nwk::try_from_hex) .transpose()?; let pan = args - .satellite_pan + .zloe_satellite_pan .as_deref() .map(PanId::try_from_hex) .transpose()?; @@ -1197,9 +1197,9 @@ fn main() -> Result<(), Box> { }); } - if let Some(coordinator) = args.satellite_of { + if let Some(coordinator) = args.zloe_satellite_of { let nwk = args - .satellite_nwk + .zloe_satellite_nwk .as_deref() .map(Nwk::try_from_hex) .transpose()?; From e43043d0877b159ac3ea854748a95f86628fb58a Mon Sep 17 00:00:00 2001 From: Igor Yamolov Date: Tue, 10 Nov 2026 14:51:46 +0100 Subject: [PATCH 07/22] SPIKE: deny experiment results, zloe-ctl lqi and latency - zloe-ctl: onoff takes an endpoint, sends with radius 30 (radius 0 was clamped to 1 and dropped at the first relay), stops at the APS ack confirm and prints elapsed time; new `lqi` decodes Mgmt_Lqi_rsp - SPIKE.md: findings of the bench runs Co-Authored-By: Claude Opus 5.5 --- crates/ziggurat-phy-trel/SPIKE.md | 38 +++++++++ crates/ziggurat-server/src/bin/zloe_ctl.rs | 94 ++++++++++++++++++++-- 2 files changed, 125 insertions(+), 7 deletions(-) diff --git a/crates/ziggurat-phy-trel/SPIKE.md b/crates/ziggurat-phy-trel/SPIKE.md index 5ab04a0..4f07fa2 100644 --- a/crates/ziggurat-phy-trel/SPIKE.md +++ b/crates/ziggurat-phy-trel/SPIKE.md @@ -71,3 +71,41 @@ interface's IPv4 address, mDNS runs only there and advertises only that address, peer addresses on the same subnet are preferred. Without it, all interfaces are used (mDNS then advertises Docker bridges and loopback too). The old `--backbone-*` flag names still work as aliases. + +## Findings (bench, 2026-11-10) + +Bench: coordinator and satellite are ESP32-C6 boards on stock OpenThread RCP (UART, no +flow control) driven by Ziggurat on one Linux host, ZLOE on `enp4s0f0`; two more +ESP32-C6 boards run an esp-zigbee-lib 2.1 (ZBOSS) on/off-light **router** +(spinel2quic `zbpeer`, 4 MB build) as third-party routers. Test network on channel 25. +Driven without ZHA: `--test-network` + `zloe-ctl`. + +- **Discovery / bootstrap:** satellite started before the coordinator, found it over + mDNS ~4 s after it came up, fetched settings over ZLOE and was a router 60 ms later. + Backbone neighbours settle at cost 1/1 within one Link Status period. +- **mDNS advertises every interface** (Docker bridges, loopback): fixed by preferring + LAN addresses and locking a peer to its datagrams' source; `--zloe-interface` confines + everything to one NIC. +- **Out-of-range via the backbone works:** with the coordinator denying router `0xBDE5`, + the coordinator source-routes through the satellite (`relays: [satellite]`), the + satellite relays over the air. On/Off: 10/10 delivered, 42–74 ms to APS ack + (median ~55 ms). Mgmt_Lqi round trip over the same path: 42 ms. +- **Direct path (no deny):** 10/10 over the air, 40–68 ms (median ~41 ms); the backbone + is not used when the destination is a direct neighbour. +- **Third-party routers are not confused:** both ZBOSS routers list the coordinator and + the satellite as ordinary sibling routers with plausible LQIs; nothing anomalous seen + in their Link Status handling. (The overlapping-Link-Status worry applies to one node + with several radios; a satellite is a separate node, so it does not arise.) +- **ESP RCP LQI is low:** boards centimetres apart report LQA ~10 (cost 7), so the + backbone's cost 1 dominates route selection. +- **Unrelated upstream issues hit on the way:** + - zigpy/ziggurat#88 (router joiners leave): worked around in the spike by delivering + to router children directly. + - Without zigpy nobody answers `Node_Desc_req`; ZBOSS joiners leave. `--test-network` + answers it. + - Host sends with radius 0 go out with radius 1 (clamped); a relayed hop then drops + them ("radius is exhausted"). zigpy always sends a real radius, `zloe-ctl` now 30. + +Not covered: satellites accepting joins (router-side join), children on satellites, +multiple satellites, Wi-Fi backbone under loss, restart/rejoin behaviour, real IKEA +devices. diff --git a/crates/ziggurat-server/src/bin/zloe_ctl.rs b/crates/ziggurat-server/src/bin/zloe_ctl.rs index aa309b7..b01aba1 100644 --- a/crates/ziggurat-server/src/bin/zloe_ctl.rs +++ b/crates/ziggurat-server/src/bin/zloe_ctl.rs @@ -1,7 +1,8 @@ //! SPIKE (throwaway): poke a Ziggurat node's API for the ZLOE bench without ZHA. //! //! zloe-ctl permit -//! zloe-ctl onoff +//! zloe-ctl onoff [endpoint, default 1] +//! zloe-ctl lqi (Mgmt_Lqi_req, decodes the response) //! zloe-ctl listen (print notifications) use futures_util::{SinkExt, StreamExt}; @@ -33,7 +34,8 @@ async fn main() -> Result<(), Error> { ); (proto::RequestCommand::PermitJoins, body) } - ("onoff", [nwk, action]) => { + ("onoff", [nwk, action, ep @ ..]) => { + let dst_ep: u8 = ep.first().map_or(Ok(1), |e| e.parse())?; let zcl_command = match action.as_str() { "off" => 0x00, "on" => 0x01, @@ -58,9 +60,9 @@ async fn main() -> Result<(), Error> { profile_id: 0x0104, cluster_id: 0x0006, src_ep: 1, - dst_ep: 1, + dst_ep, aps_seq: 0, - radius: 0, + radius: 30, priority: 0, route: proto::RouteControl::StackDecides, next_hop: None, @@ -70,6 +72,35 @@ async fn main() -> Result<(), Error> { ); (proto::RequestCommand::SendUnicast, body) } + ("lqi", [nwk]) => { + let mut body = Vec::new(); + proto::append( + &mut body, + &proto::SendUnicastPayload { + flags: proto::SendUnicastFlags { + has_eui64: false, + aps_ack: false, + aps_encryption: false, + sleepy_destination: false, + force_aps_seq: false, + }, + destination: Nwk::try_from_hex(nwk)?, + destination_eui64: Eui64([0; 8]), + profile_id: 0x0000, + cluster_id: 0x0031, + src_ep: 0, + dst_ep: 0, + aps_seq: 0, + radius: 30, + priority: 0, + route: proto::RouteControl::StackDecides, + next_hop: None, + relays: None, + asdu: vec![0x42, 0x00], + }, + ); + (proto::RequestCommand::SendUnicast, body) + } ("listen", [secs]) => { let deadline = std::time::Duration::from_secs(secs.parse()?); let _ = tokio::time::timeout(deadline, async { @@ -90,6 +121,8 @@ async fn main() -> Result<(), Error> { _ => return Err(format!("unknown command {command} {rest:?}").into()), }; + let started = std::time::Instant::now(); + let is_lqi = command == "lqi"; let mut frame = proto::envelope(proto::FrameType::Request, opcode as u8, 1); frame.extend_from_slice(&payload); ws.send(Message::Binary(frame.into())).await?; @@ -107,13 +140,28 @@ async fn main() -> Result<(), Error> { continue; }; let body = &data[consumed..]; + let ms = started.elapsed().as_millis(); + let notification = header.frame_type == proto::FrameType::Notification; + if notification && header.command == 0x10 { + if let Some(aps) = proto::parse::(body) + && aps.cluster_id == 0x8031 + { + print_lqi_rsp(&aps, ms); + if is_lqi { + break; + } + } + continue; + } println!( - "{:?} cmd={:#04x} {:02x?}", + "+{ms}ms {:?} cmd={:#04x} {:02x?}", header.frame_type, header.command, body ); - let done = header.frame_type == proto::FrameType::Response + // 0x12 is the APS ack confirm: the end of a unicast's story + let done = (header.frame_type == proto::FrameType::Response && header.command == opcode as u8 - && opcode != proto::RequestCommand::SendUnicast; + && opcode != proto::RequestCommand::SendUnicast) + || (notification && header.command == 0x12 && !is_lqi); if done { break; } @@ -122,3 +170,35 @@ async fn main() -> Result<(), Error> { .await; Ok(()) } + +/// Mgmt_Lqi_rsp: seq, status, total, start, count, then 22-byte neighbor records. +fn print_lqi_rsp(aps: &proto::ReceivedApsPayload, ms: u128) { + let d = &aps.data; + if d.len() < 5 { + return; + } + println!( + "+{ms}ms Mgmt_Lqi_rsp from {:?}: status={} total={} start={} count={}", + aps.source, d[1], d[2], d[3], d[4] + ); + for rec in d[5..].chunks_exact(22) { + let nwk = u16::from_le_bytes([rec[16], rec[17]]); + let kind = match rec[18] & 0x03 { + 0 => "coordinator", + 1 => "router", + 2 => "end device", + _ => "unknown", + }; + let relation = match (rec[18] >> 4) & 0x07 { + 0 => "parent", + 1 => "child", + 2 => "sibling", + 3 => "none", + _ => "previous child", + }; + println!( + " nwk={nwk:#06x} {kind:<11} {relation:<14} depth={} lqi={}", + rec[20], rec[21] + ); + } +} From b20a4d74e7028476961bd15c293ed47a15936091 Mon Sep 17 00:00:00 2001 From: Igor Yamolov Date: Tue, 10 Nov 2026 15:10:12 +0100 Subject: [PATCH 08/22] SPIKE: ZLOE wire v2 with extended PAN ID and IPv6 link-local - Scope peers by extended PAN ID: mDNS TXT xa/xp as in TREL, header carries the epan; --zloe-satellite-pan becomes --zloe-satellite-epan - Dual-stack UDP socket; prefer IPv6 link-local on --zloe-interface, --zloe-ipv4 to prefer IPv4; rotate through advertised addresses when sends go unanswered - 32-bit packet number, "ZL" magic, wire version 2 - Backbone LQI from ack success; drop over-the-air copies from live backbone peers instead of pinning their LQI to 255 Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 2 + crates/ziggurat-phy-trel/Cargo.toml | 2 + crates/ziggurat-phy-trel/SPIKE.md | 20 +- crates/ziggurat-phy-trel/src/lib.rs | 531 +++++++++++++++------ crates/ziggurat-phy-trel/tests/loopback.rs | 18 +- crates/ziggurat-server/src/bin/zloe_ctl.rs | 2 +- crates/ziggurat-server/src/main.rs | 21 +- crates/ziggurat-server/src/satellite.rs | 4 +- 8 files changed, 438 insertions(+), 162 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6562621..0561fed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1970,8 +1970,10 @@ dependencies = [ "if-addrs", "mdns-sd", "parking_lot", + "socket2", "tokio", "tracing", + "tracing-subscriber", "ziggurat-ieee-802154", "ziggurat-phy", ] diff --git a/crates/ziggurat-phy-trel/Cargo.toml b/crates/ziggurat-phy-trel/Cargo.toml index fbcc2cd..06e5bdf 100644 --- a/crates/ziggurat-phy-trel/Cargo.toml +++ b/crates/ziggurat-phy-trel/Cargo.toml @@ -17,6 +17,8 @@ tokio = { version = "1.53.1", features = ["rt", "time", "sync", "net", "macros"] tracing = "0.1" mdns-sd = "0.21.7" if-addrs = "0.15" +socket2 = "0.6.5" [dev-dependencies] tokio = { version = "1.53.1", features = ["rt-multi-thread", "macros", "time", "sync", "net"] } +tracing-subscriber = { version = "0.3.23", features = ["env-filter"] } diff --git a/crates/ziggurat-phy-trel/SPIKE.md b/crates/ziggurat-phy-trel/SPIKE.md index 4f07fa2..5ace0a8 100644 --- a/crates/ziggurat-phy-trel/SPIKE.md +++ b/crates/ziggurat-phy-trel/SPIKE.md @@ -17,7 +17,7 @@ coordinator over IP, extend coverage without any NWK changes? each other (mDNS TXT `role=coordinator`), fetches its settings with a `NETINFO_REQ`/`NETINFO_RSP` exchange over ZLOE and starts in router role. The backbone is treated as **trusted**: the settings, network key included, go - unauthenticated to whoever asks. `--zloe-satellite-pan ` picks a network when several + unauthenticated to whoever asks. `--zloe-satellite-epan <16 hex>` picks a network when several coordinators are on the LAN; the satellite may start before the coordinator (it waits up to 60 s). `--zloe-satellite-of ws://coord:9999` (fetch via the coordinator's Ziggurat API) remains as a fallback. NWK address derived from the @@ -109,3 +109,21 @@ Driven without ZHA: `--test-network` + `zloe-ctl`. Not covered: satellites accepting joins (router-side join), children on satellites, multiple satellites, Wi-Fi backbone under loss, restart/rejoin behaviour, real IKEA devices. + +## Wire format v2 (TREL-aligned) + +- mDNS TXT: `xa` (extended address), `xp` (extended PAN ID), `nwk`, `role`, `v=2`. + Peers on another extended PAN ID are ignored; `--zloe-satellite-epan` picks one + network when several coordinators are on the LAN. +- Header (27 bytes): `"ZL" | ver=2 | kind | flags | seq:u32le | xa | nwk:u16le | epan`. +- One dual-stack UDP socket (IPv4 arrives v4-mapped; IPv4-only fallback). With + `--zloe-interface`, peers are reached over IPv6 link-local on that interface first + (`--zloe-ipv4` prefers IPv4). Without it, link-local scopes are ambiguous (mDNS + reports one address per interface the answer arrived on), so IPv4/routable IPv6 + come first. Unanswered sends rotate through the peer's advertised addresses. +- Link quality: frames over the backbone carry an LQI from backbone ack success + (EWMA, 255 = clean); over-the-air copies of a live backbone peer's frames are + dropped instead of being re-labelled. +- Bench: bootstrap and backbone over `fe80::…%enp4s0f0`; deny run 4/4 via the + satellite, 41–64 ms. Known gap: a peer's address locks to its datagrams' source, so + `--zloe-ipv4` on one side is overridden when the other side sends over IPv6. diff --git a/crates/ziggurat-phy-trel/src/lib.rs b/crates/ziggurat-phy-trel/src/lib.rs index af3e812..b3b486e 100644 --- a/crates/ziggurat-phy-trel/src/lib.rs +++ b/crates/ziggurat-phy-trel/src/lib.rs @@ -9,12 +9,15 @@ //! failure; //! * broadcasts go over the air *and* to every backbone peer (NWK broadcast dedup //! absorbs the duplicates); -//! * frames from backbone peers are reported with LQI 255 (whichever link they came in -//! on), so NWK link costs to backbone peers are 1 and routing prefers the backbone -//! without any NWK changes. +//! * frames received over the backbone carry an LQI derived from the backbone link's +//! health (ack success), 255 when clean, so NWK link costs to backbone peers are low +//! and routing prefers the backbone without NWK changes; over-the-air copies of a +//! live backbone peer's frames are dropped (they also come over the backbone), so the +//! peer's link quality is not mixed with the radio's. //! -//! Peers are discovered with mDNS/DNS-SD (`_zloe._udp`), configured statically, or -//! learned from the first datagram they send us. +//! Peers are discovered with mDNS/DNS-SD (`_zloe._udp`, TXT `xa`/`xp` as in TREL), +//! configured statically, or learned from the first datagram they send us. Transport +//! is one dual-stack UDP socket: IPv6 link-local is preferred, IPv4 works too. //! //! A test-only deny list fakes "out of radio range": over-the-air frames whose MAC //! source is denied are dropped, and over-the-air unicasts to a denied MAC destination @@ -22,21 +25,21 @@ //! devices at the hardware level; see the spike notes. use std::collections::{HashMap, HashSet, VecDeque}; -use std::net::{IpAddr, Ipv4Addr, SocketAddr}; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, SocketAddrV6}; use std::sync::Arc; -use std::sync::atomic::{AtomicU16, Ordering}; +use std::sync::atomic::{AtomicU32, Ordering}; use std::time::{Duration, Instant}; use parking_lot::Mutex; use tokio::net::UdpSocket; use tokio::sync::{mpsc, oneshot}; -use ziggurat_ieee_802154::types::{Eui64, Nwk, PanId}; +use ziggurat_ieee_802154::types::{Eui64, Nwk}; use ziggurat_phy::{RadioConfig, RadioError, RadioPhy, Receiver, RxFrame, TxFrame, TxResult}; pub const SERVICE_TYPE: &str = "_zloe._udp.local."; -const MAGIC: [u8; 2] = *b"ZT"; -const VERSION: u8 = 1; -const HEADER_LEN: usize = 19; +const MAGIC: [u8; 2] = *b"ZL"; +const VERSION: u8 = 2; +const HEADER_LEN: usize = 27; const KIND_DATA: u8 = 0; const KIND_ACK: u8 = 1; /// Trusted-backbone bootstrap: ask a coordinator for the network settings, and its @@ -51,9 +54,13 @@ const FLAG_ACK_REQUEST: u8 = 0x01; const ACK_TIMEOUT: Duration = Duration::from_millis(50); const ATTEMPTS: usize = 3; const DEDUP_WINDOW: usize = 64; -/// LQI and RSSI reported for frames from backbone peers. -const BACKBONE_LQI: u8 = 255; +/// RSSI reported for frames received over the backbone. const BACKBONE_RSSI: i8 = -20; +/// A peer whose datagrams arrived this recently has a live backbone link: its +/// over-the-air frames are duplicates (two Link Status periods). +const BACKBONE_LIVE: Duration = Duration::from_secs(35); +/// Weight of the newest sample in the backbone ack-success average. +const HEALTH_ALPHA: f32 = 0.125; #[derive(Debug, Clone, Default)] pub struct DenyList { @@ -78,26 +85,37 @@ pub struct TrelConfig { /// Peers reachable without discovery; their identity is learned from traffic pub static_peers: Vec, pub mdns: bool, - /// Network interface (e.g. `eth0`) the backbone is confined to: the UDP socket - /// binds its IPv4 address and mDNS only runs there. `None`: all interfaces. + /// Network interface (e.g. `eth0`) the backbone is confined to: mDNS only runs + /// there and peers are reached through its addresses. `None`: all interfaces. pub interface: Option, + /// Reach peers over IPv4 when they advertise both (default: IPv6 link-local) + pub prefer_ipv4: bool, pub deny: DenyList, } -/// The IPv4 address and netmask of a named interface. -fn interface_v4(name: &str) -> std::io::Result<(Ipv4Addr, Ipv4Addr)> { - if_addrs::get_if_addrs()? +/// The addresses of the interface the backbone is confined to. +#[derive(Debug, Clone, Copy)] +struct LocalNet { + index: u32, + v4: Option<(Ipv4Addr, Ipv4Addr)>, +} + +fn local_net(name: &str) -> std::io::Result { + let addrs: Vec<_> = if_addrs::get_if_addrs()? .into_iter() - .find_map(|iface| match iface.addr { - if_addrs::IfAddr::V4(v4) if iface.name == name => Some((v4.ip, v4.netmask)), - _ => None, - }) - .ok_or_else(|| { - std::io::Error::new( - std::io::ErrorKind::NotFound, - format!("ZLOE interface {name:?} has no IPv4 address"), - ) - }) + .filter(|iface| iface.name == name) + .collect(); + let index = addrs.iter().find_map(|i| i.index).ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::NotFound, + format!("ZLOE interface {name:?} not found"), + ) + })?; + let v4 = addrs.iter().find_map(|i| match &i.addr { + if_addrs::IfAddr::V4(v4) => Some((v4.ip, v4.netmask)), + if_addrs::IfAddr::V6(_) => None, + }); + Ok(LocalNet { index, v4 }) } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -112,28 +130,82 @@ struct Peer { nwk: Option, source: PeerSource, last_rx: Option, + /// Backbone ack success, exponentially averaged (1.0 = clean) + health: f32, + /// Every usable address mDNS gave, best first: tried in turn when sends fail + alternates: Vec, } +impl Peer { + fn new(addr: SocketAddr, nwk: Option, source: PeerSource) -> Self { + Self { + addr, + nwk, + source, + last_rx: None, + health: 1.0, + alternates: Vec::new(), + } + } + + /// The current address went unanswered: move to the next advertised one. + fn rotate(&mut self) { + if self.alternates.len() < 2 { + return; + } + let next = self + .alternates + .iter() + .position(|a| *a == self.addr) + .map_or(0, |i| (i + 1) % self.alternates.len()); + tracing::info!( + "ZLOE: {} unanswered, trying {}", + self.addr, + self.alternates[next] + ); + self.addr = self.alternates[next]; + } + + fn live(&self) -> bool { + self.last_rx.is_some_and(|t| t.elapsed() < BACKBONE_LIVE) + } + + fn lqi(&self) -> u8 { + (self.health * 255.0).round().clamp(1.0, 255.0) as u8 + } + + fn record(&mut self, acked: bool) { + let sample = if acked { 1.0 } else { 0.0 }; + self.health += HEALTH_ALPHA * (sample - self.health); + } +} + +/// Our identity on the backbone: complete once both the radio configuration (address, +/// channel) and the network's extended PAN ID are known. #[derive(Debug, Clone, Copy)] struct Identity { eui64: Eui64, nwk: Nwk, - pan_id: PanId, + epan: Eui64, channel: u8, } #[derive(Default)] struct Shared { identity: Option, + /// Radio half of the identity, from `reconfigure` + radio: Option<(Eui64, Nwk, u8)>, + /// Network half, from `set_extended_pan_id` + epan: Option, peers: HashMap, /// Recently delivered (sender, backbone seq), to drop retransmits of a frame whose /// ack was lost. - seen: HashMap>, + seen: HashMap>, rx_tx: Option>, mdns_fullname: Option, /// Coordinators seen on the backbone (mDNS `role=coordinator`), kept even before /// we have an identity of our own: satellites bootstrap from them - coordinators: HashMap)>, + coordinators: HashMap)>, netinfo_provider: Option, netinfo_waiter: Option>>, } @@ -141,13 +213,16 @@ struct Shared { struct Backbone { socket: Arc, shared: Arc>, - pending: Arc>>>, - seq: AtomicU16, + pending: Arc>>>, + seq: AtomicU32, static_peers: Vec, mdns: Option, port: u16, - /// Set when confined to one interface: its address and netmask - local_net: Option<(Ipv4Addr, Ipv4Addr)>, + /// Set when confined to one interface + local_net: Option, + prefer_ipv4: bool, + /// The socket is IPv6 dual-stack: IPv4 targets go out as v4-mapped addresses + dual_stack: bool, } pub struct TrelPhy { @@ -174,13 +249,23 @@ impl TrelPhy

{ let backbone = if config.port == 0 { None } else { - let local_net = config.interface.as_deref().map(interface_v4).transpose()?; - let bind_ip = local_net.map_or(Ipv4Addr::UNSPECIFIED, |(ip, _)| ip); - let socket = Arc::new(UdpSocket::bind((bind_ip, config.port)).await?); + let local_net = config.interface.as_deref().map(local_net).transpose()?; + let (socket, dual_stack) = bind_socket(config.port)?; + let socket = Arc::new(socket); tracing::info!( - "ZLOE backbone listening on {bind_ip}:{} ({})", + "ZLOE backbone listening on UDP {} ({}, {}, prefer {})", config.port, - config.interface.as_deref().unwrap_or("all interfaces") + if dual_stack { + "IPv6 dual-stack" + } else { + "IPv4 only" + }, + config.interface.as_deref().unwrap_or("all interfaces"), + if config.prefer_ipv4 { + "IPv4" + } else { + "IPv6 link-local" + }, ); let mdns = if config.mdns { @@ -189,6 +274,7 @@ impl TrelPhy

{ if let Some(name) = &config.interface { let _ = daemon.disable_interface(mdns_sd::IfKind::All); let _ = daemon.disable_interface(mdns_sd::IfKind::LoopbackV4); + let _ = daemon.disable_interface(mdns_sd::IfKind::LoopbackV6); if let Err(e) = daemon.enable_interface(mdns_sd::IfKind::Name(name.clone())) { @@ -210,11 +296,13 @@ impl TrelPhy

{ socket, shared: Arc::clone(&shared), pending: Arc::new(Mutex::new(HashMap::new())), - seq: AtomicU16::new(rand_u16()), + seq: AtomicU32::new(rand_u32()), static_peers: config.static_peers, mdns, port: config.port, local_net, + prefer_ipv4: config.prefer_ipv4, + dual_stack, }; backbone.spawn_udp_reader(); backbone.spawn_mdns_browser(); @@ -233,16 +321,29 @@ impl TrelPhy

{ &self.inner } + /// The network's extended PAN ID (TREL's `xp` in mDNS): peers on other networks are + /// ignored. Set by the stack's owner when the network is configured. + pub fn set_extended_pan_id(&self, epan: Eui64) { + let identity = { + let mut shared = self.shared.lock(); + shared.epan = Some(epan); + update_identity(&mut shared) + }; + if let (Some(identity), Some(backbone)) = (identity, &self.backbone) { + backbone.advertise(identity); + } + } + /// Coordinator side: hand these settings to satellites that ask over the backbone. pub fn set_netinfo_provider(&self, provider: NetInfoProvider) { self.shared.lock().netinfo_provider = Some(provider); } - /// Satellite side: find a coordinator on the backbone (on `pan`, when given) and - /// fetch its network settings. Retries until `timeout`. + /// Satellite side: find a coordinator on the backbone (on extended PAN `epan`, when + /// given) and fetch its network settings. Retries until `timeout`. pub async fn fetch_network( &self, - pan: Option, + epan: Option, timeout: Duration, ) -> std::io::Result> { let backbone = self @@ -252,23 +353,23 @@ impl TrelPhy

{ let deadline = tokio::time::Instant::now() + timeout; while tokio::time::Instant::now() < deadline { - let candidates: Vec<(Eui64, SocketAddr, Option)> = { + let candidates: Vec<(Eui64, SocketAddr, Option)> = { let shared = self.shared.lock(); shared .coordinators .iter() - .filter(|(_, (_, p))| pan.is_none() || *p == pan) - .map(|(eui64, (addr, p))| (*eui64, *addr, *p)) + .filter(|(_, (_, x))| epan.is_none() || *x == epan) + .map(|(eui64, (addr, x))| (*eui64, *addr, *x)) .collect() }; - let Some(&(eui64, addr, coordinator_pan)) = candidates.first() else { + let Some(&(eui64, addr, coordinator_epan)) = candidates.first() else { tokio::time::sleep(Duration::from_millis(250)).await; continue; }; if candidates.len() > 1 { tracing::warn!( - "ZLOE: {} coordinators match, using {eui64:?} (PAN {coordinator_pan:?}); \ - pick one with --satellite-pan", + "ZLOE: {} coordinators match, using {eui64:?} (epan {coordinator_epan:?}); \ + pick one with --zloe-satellite-epan", candidates.len() ); } @@ -278,7 +379,10 @@ impl TrelPhy

{ let seq = backbone.seq.fetch_add(1, Ordering::Relaxed); let request = netinfo_header(KIND_NETINFO_REQ, seq, None); tracing::info!("ZLOE: asking coordinator {eui64:?} at {addr} for network settings"); - backbone.socket.send_to(&request, addr).await?; + backbone + .socket + .send_to(&request, backbone.target(addr)) + .await?; if let Ok(Ok(info)) = tokio::time::timeout(Duration::from_secs(1), rx).await { return Ok(info); } @@ -291,29 +395,80 @@ impl TrelPhy

{ } /// Header for the bootstrap messages, sent before we may have an identity. -fn netinfo_header(kind: u8, seq: u16, identity: Option) -> Vec { +fn netinfo_header(kind: u8, seq: u32, identity: Option) -> Vec { WireHeader { kind, flags: 0, seq, eui64: identity.map_or(Eui64([0; 8]), |i| i.eui64), nwk: identity.map_or(Nwk(0xFFFF), |i| i.nwk), - pan_id: identity.map_or(PanId(0xFFFF), |i| i.pan_id), + epan: identity.map_or(Eui64([0; 8]), |i| i.epan), } .to_bytes() } +/// Combine the radio and network halves into the identity, once both are known. +fn update_identity(shared: &mut Shared) -> Option { + let (eui64, nwk, channel) = shared.radio?; + let epan = shared.epan?; + let identity = Identity { + eui64, + nwk, + epan, + channel, + }; + let changed = shared.identity.is_none_or(|old| { + old.eui64 != identity.eui64 || old.nwk != identity.nwk || old.epan != identity.epan + }); + shared.identity = Some(identity); + changed.then_some(identity) +} + +/// One dual-stack IPv6 socket (IPv4 arrives v4-mapped); IPv4-only when the host has no +/// IPv6. +fn bind_socket(port: u16) -> std::io::Result<(UdpSocket, bool)> { + use socket2::{Domain, Protocol, Socket, Type}; + let v6 = Socket::new(Domain::IPV6, Type::DGRAM, Some(Protocol::UDP)).and_then(|s| { + s.set_only_v6(false)?; + s.bind(&SocketAddr::from((Ipv6Addr::UNSPECIFIED, port)).into())?; + Ok(s) + }); + let (socket, dual_stack) = match v6 { + Ok(s) => (s, true), + Err(e) => { + tracing::warn!("ZLOE: no IPv6 socket ({e}), using IPv4 only"); + let s = Socket::new(Domain::IPV4, Type::DGRAM, Some(Protocol::UDP))?; + s.bind(&SocketAddr::from((Ipv4Addr::UNSPECIFIED, port)).into())?; + (s, false) + } + }; + socket.set_nonblocking(true)?; + Ok((UdpSocket::from_std(socket.into())?, dual_stack)) +} + +/// v4-mapped IPv6 back to plain IPv4, so a peer has one spelling of its address. +fn canonical(addr: SocketAddr) -> SocketAddr { + match addr { + SocketAddr::V6(v6) => v6 + .ip() + .to_ipv4_mapped() + .map_or(addr, |v4| SocketAddr::new(IpAddr::V4(v4), v6.port())), + SocketAddr::V4(_) => addr, + } +} + impl Backbone { fn spawn_udp_reader(&self) { let socket = Arc::clone(&self.socket); let shared = Arc::clone(&self.shared); let pending = Arc::clone(&self.pending); + let dual_stack = self.dual_stack; tokio::spawn(async move { let mut buf = vec![0u8; 2048]; loop { let (len, from) = match socket.recv_from(&mut buf).await { - Ok(r) => r, + Ok((len, from)) => (len, canonical(from)), Err(e) => { tracing::warn!("ZLOE: UDP receive failed: {e}"); tokio::time::sleep(Duration::from_millis(100)).await; @@ -348,7 +503,7 @@ impl Backbone { ); let mut rsp = netinfo_header(KIND_NETINFO_RSP, header.seq, identity); rsp.extend_from_slice(&info); - let _ = socket.send_to(&rsp, from).await; + let _ = socket.send_to(&rsp, to_family(from, dual_stack)).await; continue; } if header.kind == KIND_NETINFO_RSP { @@ -364,7 +519,7 @@ impl Backbone { let Some(identity) = shared.identity else { continue; }; - if header.pan_id != identity.pan_id || header.eui64 == identity.eui64 { + if header.epan != identity.epan || header.eui64 == identity.eui64 { continue; } @@ -375,16 +530,12 @@ impl Backbone { header.eui64, header.nwk ); - Peer { - addr: from, - nwk: None, - source: PeerSource::Learned, - last_rx: None, - } + Peer::new(from, None, PeerSource::Learned) }); peer.addr = from; peer.nwk = Some(header.nwk); peer.last_rx = Some(Instant::now()); + let lqi = peer.lqi(); let seen = shared.seen.entry(header.eui64).or_default(); let duplicate = seen.contains(&header.seq); @@ -403,7 +554,7 @@ impl Backbone { psdu, channel: identity.channel, rssi: BACKBONE_RSSI, - lqi: BACKBONE_LQI, + lqi, timestamp_us: 0, }, ) @@ -418,10 +569,10 @@ impl Backbone { seq: header.seq, eui64: identity.eui64, nwk: identity.nwk, - pan_id: identity.pan_id, + epan: identity.epan, } .to_bytes(); - let _ = socket.send_to(&ack, to).await; + let _ = socket.send_to(&ack, to_family(to, dual_stack)).await; } if let Some((Some(tx), frame)) = deliver { @@ -447,63 +598,68 @@ impl Backbone { }; let shared = Arc::clone(&self.shared); let local_net = self.local_net; + let prefer_ipv4 = self.prefer_ipv4; tokio::spawn(async move { while let Ok(event) = events.recv_async().await { match event { mdns_sd::ServiceEvent::ServiceResolved(info) => { let Some(eui64) = info - .get_property_val_str("eui64") + .get_property_val_str("xa") .and_then(|s| Eui64::try_from_hex(s).ok()) else { continue; }; - let pan_id = info - .get_property_val_str("pan") - .and_then(|s| PanId::try_from_hex(s).ok()); + let epan = info + .get_property_val_str("xp") + .and_then(|s| Eui64::try_from_hex(s).ok()); let nwk = info .get_property_val_str("nwk") .and_then(|s| Nwk::try_from_hex(s).ok()); - // mDNS lists every interface (Docker bridges, loopback, ...): - // pick deterministically, LAN first - let Some(ip) = info.get_addresses_v4().into_iter().min_by_key(|ip| { - let on_our_net = local_net.is_some_and(|(ours, mask)| { - u32::from(*ip) & u32::from(mask) - == u32::from(ours) & u32::from(mask) - }); - (!on_our_net, address_rank(*ip), *ip) - }) else { + // mDNS lists every address of every interface (Docker bridges, + // loopback, ...): pick deterministically + let candidates = candidate_addresses( + info.get_addresses(), + info.get_port(), + local_net, + prefer_ipv4, + ); + let Some(&(best_key, addr)) = candidates.first() else { continue; }; - let addr = SocketAddr::new(IpAddr::V4(ip), info.get_port()); + let alternates: Vec = + candidates.iter().map(|(_, a)| *a).collect(); let mut shared = shared.lock(); if info.get_property_val_str("role") == Some("coordinator") { - let known = shared.coordinators.insert(eui64, (addr, pan_id)); + let known = shared.coordinators.insert(eui64, (addr, epan)); if known.is_none() { tracing::info!( - "ZLOE: coordinator {eui64:?} (PAN {pan_id:?}) at {addr}" + "ZLOE: coordinator {eui64:?} (epan {epan:?}) at {addr}" ); } } let Some(identity) = shared.identity else { continue; }; - if eui64 == identity.eui64 || pan_id != Some(identity.pan_id) { + if eui64 == identity.eui64 || epan != Some(identity.epan) { continue; } let peer = shared.peers.entry(eui64).or_insert_with(|| { tracing::info!("ZLOE: mDNS peer {eui64:?} / {nwk:?} at {addr}"); - Peer { - addr, - nwk, - source: PeerSource::Mdns, - last_rx: None, - } + Peer::new(addr, nwk, PeerSource::Mdns) }); // Once the peer has talked to us, its datagrams' source address // is authoritative - if peer.last_rx.is_none() && peer.addr != addr { + // Until the peer talks to us, follow mDNS, but only to a + // strictly better address: answers trickle in per interface + // and would make it flap + let current_key = candidates + .iter() + .find(|(_, a)| *a == peer.addr) + .map(|(k, _)| *k); + peer.alternates = alternates; + if peer.last_rx.is_none() && current_key.is_none_or(|k| best_key < k) { tracing::info!("ZLOE: mDNS peer {eui64:?} moved to {addr}"); peer.addr = addr; } @@ -545,9 +701,9 @@ impl Backbone { let instance = hex_be(&identity.eui64.to_be_bytes()); let properties: HashMap = [ ("v".to_string(), VERSION.to_string()), - ("eui64".to_string(), instance.clone()), + ("xa".to_string(), instance.clone()), + ("xp".to_string(), hex_be(&identity.epan.to_be_bytes())), ("nwk".to_string(), format!("{:04x}", identity.nwk.as_u16())), - ("pan".to_string(), format!("{:04x}", identity.pan_id.0)), ( "role".to_string(), if identity.nwk == Nwk(0x0000) { @@ -561,25 +717,12 @@ impl Backbone { .into_iter() .collect(); + // Addresses (IPv6 link-local and IPv4) come from the interfaces mDNS runs on, + // which --zloe-interface already narrows to one let host = format!("ziggurat-{instance}.local."); - let ip = self - .local_net - .map(|(ip, _)| ip.to_string()) - .unwrap_or_default(); - match mdns_sd::ServiceInfo::new( - SERVICE_TYPE, - &instance, - &host, - ip.as_str(), - self.port, - properties, - ) { + match mdns_sd::ServiceInfo::new(SERVICE_TYPE, &instance, &host, "", self.port, properties) { Ok(info) => { - let info = if self.local_net.is_some() { - info - } else { - info.enable_addr_auto() - }; + let info = info.enable_addr_auto(); let fullname = info.get_fullname().to_string(); match daemon.register(info) { Ok(()) => { @@ -618,7 +761,7 @@ impl Backbone { } } - fn header(&self, kind: u8, flags: u8, seq: u16) -> Option> { + fn header(&self, kind: u8, flags: u8, seq: u32) -> Option> { let identity = self.shared.lock().identity?; Some( WireHeader { @@ -627,12 +770,24 @@ impl Backbone { seq, eui64: identity.eui64, nwk: identity.nwk, - pan_id: identity.pan_id, + epan: identity.epan, } .to_bytes(), ) } + /// The address as this socket's family wants it. + fn target(&self, addr: SocketAddr) -> SocketAddr { + to_family(addr, self.dual_stack) + } + + fn record_link(&self, to: SocketAddr, acked: bool) { + let mut shared = self.shared.lock(); + if let Some(peer) = shared.peers.values_mut().find(|p| p.addr == to) { + peer.record(acked); + } + } + async fn send_broadcast(&self, psdu: &[u8]) { let seq = self.seq.fetch_add(1, Ordering::Relaxed); let Some(mut datagram) = self.header(KIND_DATA, 0, seq) else { @@ -640,7 +795,7 @@ impl Backbone { }; datagram.extend_from_slice(psdu); for target in self.broadcast_targets() { - let _ = self.socket.send_to(&datagram, target).await; + let _ = self.socket.send_to(&datagram, self.target(target)).await; } } @@ -653,17 +808,21 @@ impl Backbone { }; datagram.extend_from_slice(psdu); + let dest = self.target(to); if !ack_request { - return self.socket.send_to(&datagram, to).await.is_ok(); + return self.socket.send_to(&datagram, dest).await.is_ok(); } for attempt in 0..ATTEMPTS { let (tx, rx) = oneshot::channel(); self.pending.lock().insert(seq, tx); - if self.socket.send_to(&datagram, to).await.is_err() { + if self.socket.send_to(&datagram, dest).await.is_err() { + self.record_link(to, false); break; } - if tokio::time::timeout(ACK_TIMEOUT, rx).await.is_ok() { + let acked = tokio::time::timeout(ACK_TIMEOUT, rx).await.is_ok(); + self.record_link(to, acked); + if acked { if attempt > 0 { tracing::debug!("ZLOE: unicast to {to} acked after {} retries", attempt); } @@ -672,6 +831,9 @@ impl Backbone { } self.pending.lock().remove(&seq); tracing::debug!("ZLOE: unicast to {to} not acked"); + if let Some(peer) = self.shared.lock().peers.values_mut().find(|p| p.addr == to) { + peer.rotate(); + } false } } @@ -692,23 +854,16 @@ impl RadioPhy for TrelPhy

{ if config.short_address == Nwk(0xFFFF) || config.promiscuous { return Ok(()); } - let identity = Identity { - eui64: config.extended_address, - nwk: config.short_address, - pan_id: config.pan_id, - channel: config.channel, - }; - let changed = { + let identity = { let mut shared = self.shared.lock(); - let changed = shared.identity.is_none_or(|old| { - old.eui64 != identity.eui64 - || old.nwk != identity.nwk - || old.pan_id != identity.pan_id - }); - shared.identity = Some(identity); - changed + shared.radio = Some(( + config.extended_address, + config.short_address, + config.channel, + )); + update_identity(&mut shared) }; - if changed && let Some(backbone) = &self.backbone { + if let (Some(identity), Some(backbone)) = (identity, &self.backbone) { backbone.advertise(identity); } Ok(()) @@ -774,28 +929,32 @@ impl RadioPhy for TrelPhy

{ let shared = Arc::clone(&self.shared); let has_backbone = self.backbone.is_some(); - // Over-the-air frames: apply the deny list and pin peers' LQI + // Over-the-air frames: apply the deny list, and drop copies of what a live + // backbone peer also sends over the backbone tokio::spawn(async move { - while let Some(mut frame) = inner_rx.recv().await { + while let Some(frame) = inner_rx.recv().await { if let Some(mac) = MacHeader::parse(&frame.psdu) { if deny.denies(&mac.source) { tracing::trace!("ZLOE deny: dropping air frame from {:?}", mac.source); continue; } if has_backbone { - let is_peer = { + let live_peer = { let shared = shared.lock(); match &mac.source { - MacAddress::Extended(eui64) => shared.peers.contains_key(eui64), - MacAddress::Short(nwk) => { - shared.peers.values().any(|p| p.nwk == Some(*nwk)) + MacAddress::Extended(eui64) => { + shared.peers.get(eui64).is_some_and(Peer::live) } + MacAddress::Short(nwk) => shared + .peers + .values() + .any(|p| p.nwk == Some(*nwk) && p.live()), MacAddress::None => false, } }; - if is_peer { - frame.lqi = BACKBONE_LQI; - frame.rssi = BACKBONE_RSSI; + if live_peer { + tracing::trace!("ZLOE: dropping air copy from {:?}", mac.source); + continue; } } } @@ -822,10 +981,11 @@ impl TrelPhy

{ .iter() .map(|(eui64, p)| { format!( - "{eui64:?} nwk={:?} addr={} via={:?} last_rx={:?}", + "{eui64:?} nwk={:?} addr={} via={:?} lqi={} last_rx={:?}", p.nwk, p.addr, p.source, + p.lqi(), p.last_rx.map(|t| t.elapsed()) ) }) @@ -835,14 +995,15 @@ impl TrelPhy

{ // -- wire format --------------------------------------------------------------------- -/// `"ZT" | ver | kind | flags | seq:u16le | eui64 (wire order) | nwk:u16le | pan:u16le` +/// `"ZL" | ver=2 | kind | flags | seq:u32le | xa: EUI64 | nwk:u16le | epan: extended PAN ID` +/// (EUI64s in 802.15.4 wire order, i.e. little-endian) struct WireHeader { kind: u8, flags: u8, - seq: u16, + seq: u32, eui64: Eui64, nwk: Nwk, - pan_id: PanId, + epan: Eui64, } impl WireHeader { @@ -853,10 +1014,10 @@ impl WireHeader { Some(Self { kind: bytes[3], flags: bytes[4], - seq: u16::from_le_bytes([bytes[5], bytes[6]]), - eui64: Eui64(bytes[7..15].try_into().ok()?), - nwk: Nwk(u16::from_le_bytes([bytes[15], bytes[16]])), - pan_id: PanId(u16::from_le_bytes([bytes[17], bytes[18]])), + seq: u32::from_le_bytes(bytes[5..9].try_into().ok()?), + eui64: Eui64(bytes[9..17].try_into().ok()?), + nwk: Nwk(u16::from_le_bytes([bytes[17], bytes[18]])), + epan: Eui64(bytes[19..27].try_into().ok()?), }) } @@ -869,7 +1030,7 @@ impl WireHeader { out.extend_from_slice(&self.seq.to_le_bytes()); out.extend_from_slice(&self.eui64.0); out.extend_from_slice(&self.nwk.0.to_le_bytes()); - out.extend_from_slice(&self.pan_id.0.to_le_bytes()); + out.extend_from_slice(&self.epan.0); out } } @@ -935,6 +1096,76 @@ impl MacHeader { } } +/// Every address mDNS resolved for a peer that we could use, best first. +/// +/// Confined to one interface (`--zloe-interface`): IPv6 link-local on that interface +/// first (IPv4 first with `prefer_ipv4`), then IPv4 on its subnet, then routable IPv6. +/// Not confined: a link-local address's scope is ambiguous (mDNS reports it once per +/// interface the answer arrived on, Docker bridges and veths included), so IPv4 and +/// routable IPv6 come first and link-local is a last resort. +fn candidate_addresses( + addrs: &HashSet, + port: u16, + local_net: Option, + prefer_ipv4: bool, +) -> Vec<((u8, u8), SocketAddr)> { + let mut ranked: Vec<((u8, u8), SocketAddr)> = addrs + .iter() + .filter_map(|scoped| match scoped { + mdns_sd::ScopedIp::V6(v6) => { + let ip = *v6.addr(); + let index = v6.scope_id().index; + if ip.is_loopback() { + return None; + } + let link_local = (ip.segments()[0] & 0xffc0) == 0xfe80; + let class = match (local_net, link_local) { + (Some(net), true) if net.index != index => return None, + (Some(_), true) => u8::from(prefer_ipv4), + (Some(_), false) | (None, false) => 2, + (None, true) => 3, + }; + let scope = if link_local { index } else { 0 }; + Some(( + (class, 0), + SocketAddr::V6(SocketAddrV6::new(ip, port, 0, scope)), + )) + } + mdns_sd::ScopedIp::V4(v4) => { + let ip = *v4.addr(); + if let Some(net) = local_net { + let (ours, mask) = net.v4?; + if u32::from(ip) & u32::from(mask) != u32::from(ours) & u32::from(mask) { + return None; + } + } + let class = if local_net.is_some() { + u8::from(!prefer_ipv4) + } else { + 0 + }; + Some(( + (class, address_rank(ip)), + SocketAddr::new(IpAddr::V4(ip), port), + )) + } + _ => None, + }) + .collect(); + ranked.sort(); + ranked +} + +/// IPv4 targets on a dual-stack IPv6 socket must be v4-mapped. +fn to_family(addr: SocketAddr, dual_stack: bool) -> SocketAddr { + match addr { + SocketAddr::V4(v4) if dual_stack => { + SocketAddr::V6(SocketAddrV6::new(v4.ip().to_ipv6_mapped(), v4.port(), 0, 0)) + } + _ => addr, + } +} + /// Lower is better: 192.168/16, 10/8, other non-loopback, 172.16/12 (usually /// container bridges), link-local, loopback. const fn address_rank(ip: std::net::Ipv4Addr) -> u8 { @@ -953,10 +1184,10 @@ fn hex_be(bytes: &[u8]) -> String { bytes.iter().map(|b| format!("{b:02x}")).collect() } -fn rand_u16() -> u16 { +fn rand_u32() -> u32 { let nanos = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.subsec_nanos()) + .map(|d| d.as_nanos()) .unwrap_or(0); - (nanos ^ (nanos >> 16)) as u16 + (nanos ^ (nanos >> 32)) as u32 } diff --git a/crates/ziggurat-phy-trel/tests/loopback.rs b/crates/ziggurat-phy-trel/tests/loopback.rs index 7a681ef..7612c92 100644 --- a/crates/ziggurat-phy-trel/tests/loopback.rs +++ b/crates/ziggurat-phy-trel/tests/loopback.rs @@ -118,12 +118,14 @@ async fn node( }, mdns, interface: None, + prefer_ipv4: false, deny: DenyList::default(), }, ) .await .unwrap(); phy.reconfigure(&config(eui, nwk)).await.unwrap(); + phy.set_extended_pan_id(Eui64([0x5A; 8])); (phy, radio) } @@ -168,6 +170,10 @@ async fn static_peers_broadcast_then_unicast() { #[tokio::test(flavor = "multi_thread")] async fn mdns_discovery() { + let _ = tracing_subscriber::fmt() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) + .with_test_writer() + .try_init(); let (a, _) = node(47201, 0, 0xA1, 0x0000, true).await; let (b, _) = node(47202, 0, 0xB1, 0x2222, true).await; let mut rx_b = b.subscribe_rx(); @@ -188,8 +194,16 @@ async fn mdns_discovery() { ); tokio::time::sleep(Duration::from_millis(100)).await; } - let result = a.transmit(data_frame(0x2222, 0x0000, true)).await.unwrap(); - assert_eq!(result, TxResult::Acked); + // The first address mDNS offers may be a link-local one with the wrong scope; an + // unanswered send rotates to the next advertised address, as NWK retries would + let mut result = TxResult::NoAck; + for _ in 0..5 { + result = a.transmit(data_frame(0x2222, 0x0000, true)).await.unwrap(); + if result == TxResult::Acked { + break; + } + } + assert_eq!(result, TxResult::Acked, "peers: {:?}", a.peers()); tokio::time::timeout(Duration::from_secs(1), rx_b.recv()) .await .unwrap() diff --git a/crates/ziggurat-server/src/bin/zloe_ctl.rs b/crates/ziggurat-server/src/bin/zloe_ctl.rs index b01aba1..0a9f535 100644 --- a/crates/ziggurat-server/src/bin/zloe_ctl.rs +++ b/crates/ziggurat-server/src/bin/zloe_ctl.rs @@ -181,7 +181,7 @@ fn print_lqi_rsp(aps: &proto::ReceivedApsPayload, ms: u128) { "+{ms}ms Mgmt_Lqi_rsp from {:?}: status={} total={} start={} count={}", aps.source, d[1], d[2], d[3], d[4] ); - for rec in d[5..].chunks_exact(22) { + for rec in d[5..].as_chunks::<22>().0 { let nwk = u16::from_le_bytes([rec[16], rec[17]]); let kind = match rec[18] & 0x03 { 0 => "coordinator", diff --git a/crates/ziggurat-server/src/main.rs b/crates/ziggurat-server/src/main.rs index e3db058..8286af0 100644 --- a/crates/ziggurat-server/src/main.rs +++ b/crates/ziggurat-server/src/main.rs @@ -687,6 +687,8 @@ impl ZigguratServer { let phy = self.phy().await.map_err(radio_error)?; let aps_frame_counter = payload.state.aps_frame_counter; + // SPIKE: ZLOE peers are scoped by the extended PAN ID (TREL's `xp`) + phy.set_extended_pan_id(payload.state.extended_pan_id); let stack = ZigbeeStack::new( phy, proto::network_config(&payload), @@ -1040,6 +1042,11 @@ struct Args { #[arg(long, alias = "backbone-peer")] zloe_peer: Vec, + /// SPIKE: reach ZLOE peers over IPv4 when they advertise both (default: IPv6 + /// link-local) + #[arg(long)] + zloe_ipv4: bool, + /// SPIKE: disable mDNS discovery of ZLOE peers #[arg(long, alias = "no-mdns")] zloe_no_mdns: bool, @@ -1068,9 +1075,10 @@ struct Args { #[arg(long)] zloe_satellite: bool, - /// SPIKE: with --zloe-satellite, only use a coordinator on this PAN (hex) + /// SPIKE: with --zloe-satellite, only use a coordinator on this extended PAN ID + /// (16 hex digits) #[arg(long)] - zloe_satellite_pan: Option, + zloe_satellite_epan: Option, /// SPIKE: the satellite's NWK address (hex); derived from its EUI64 by default #[arg(long)] @@ -1131,6 +1139,7 @@ fn main() -> Result<(), Box> { static_peers: args.zloe_peer, mdns: !args.zloe_no_mdns, interface: args.zloe_interface, + prefer_ipv4: args.zloe_ipv4, deny, }, )); @@ -1183,14 +1192,14 @@ fn main() -> Result<(), Box> { .as_deref() .map(Nwk::try_from_hex) .transpose()?; - let pan = args - .zloe_satellite_pan + let epan = args + .zloe_satellite_epan .as_deref() - .map(PanId::try_from_hex) + .map(Eui64::try_from_hex) .transpose()?; let server = server.clone(); tokio::spawn(async move { - if let Err(e) = satellite::bootstrap_zloe(&server, pan, nwk).await { + if let Err(e) = satellite::bootstrap_zloe(&server, epan, nwk).await { tracing::error!("Satellite bootstrap failed: {e}"); std::process::exit(1); } diff --git a/crates/ziggurat-server/src/satellite.rs b/crates/ziggurat-server/src/satellite.rs index c197f9c..df75ead 100644 --- a/crates/ziggurat-server/src/satellite.rs +++ b/crates/ziggurat-server/src/satellite.rs @@ -23,13 +23,13 @@ pub async fn bootstrap( /// Find the coordinator on the (trusted) ZLOE backbone and fetch the settings from it. pub async fn bootstrap_zloe( server: &std::sync::Arc, - pan: Option, + epan: Option, nwk: Option, ) -> Result<(), Box> { tracing::info!("Satellite: looking for a coordinator on the ZLOE backbone"); let phy = server.phy().await?; let bytes = phy - .fetch_network(pan, std::time::Duration::from_secs(60)) + .fetch_network(epan, std::time::Duration::from_secs(60)) .await?; let info: proto::NetworkInfoPayload = proto::parse(&bytes).ok_or("malformed network settings from coordinator")?; From 2d1458ee94386afd8458997455ad895a4185af75 Mon Sep 17 00:00:00 2001 From: Igor Yamolov Date: Tue, 10 Nov 2026 15:32:37 +0100 Subject: [PATCH 09/22] SPIKE: design and gap analysis for ZLOE broadcast delivery Multicast as the primary broadcast transport, a hub ("hybrid star") fallback through the coordinator, direct fan-out as the last resort, and optional suppression of re-sends into the backbone. Lists the gaps of each part against the stack's passive-ack, route-request and neighbour-ageing behaviour. Design only; nothing is built. Co-Authored-By: Claude --- crates/ziggurat-phy-trel/BROADCAST.md | 220 ++++++++++++++++++++++++++ 1 file changed, 220 insertions(+) create mode 100644 crates/ziggurat-phy-trel/BROADCAST.md diff --git a/crates/ziggurat-phy-trel/BROADCAST.md b/crates/ziggurat-phy-trel/BROADCAST.md new file mode 100644 index 0000000..aab5750 --- /dev/null +++ b/crates/ziggurat-phy-trel/BROADCAST.md @@ -0,0 +1,220 @@ +# ZLOE broadcast delivery: design and gap analysis + +Status: design for the spike (zigpy/ziggurat#84). Nothing here is built yet. + +Styleguide: the repository has no written styleguide; its conventions are enforced by +`.pre-commit-config.yaml` (`cargo fmt --all`, `cargo clippy --all-targets +--all-features -- -D warnings -Dclippy::all -Dclippy::nursery`). The spike follows +them. Tests stay at spike level (loopback smoke tests plus bench runs). + +## Goal + +Today a backbone node sends every 802.15.4 broadcast as one acked unicast datagram per +peer: N×(N−1) datagrams per flooded broadcast with N backbone nodes. The goal is fewer +datagrams without changing what the Zigbee stack sees, and without requiring multicast +to work. + +## Algorithm + +One rule: **every broadcast frame a backbone node transmits reaches every other +backbone node once, as that node's own frame** (802.15.4 source unchanged). The stack +then behaves exactly as it does on a radio channel: Link Status makes backbone nodes +neighbours, route requests are heard "directly", relays are real passive acks. + +Three transports deliver that rule: + +1. **Multicast** (primary): one unacknowledged datagram to the network's IPv6 + link-local group. N datagrams per flood (one per relaying node). +2. **Hub** (fallback, "hybrid star"): a satellite sends one acked datagram to the + coordinator; the coordinator forwards it unchanged to every other backbone node. + Unicast frames stay peer-to-peer. +3. **Direct** (last resort, what exists today): one acked datagram to each peer. Used + when the coordinator does not answer. + +Selection is per sender: a node uses multicast alone while every known peer keeps +acknowledging its multicast probes; otherwise it also uses the hub; if the hub does +not acknowledge, it sends directly. + +**Optional suppression** (the "do not re-send to backbone routers" rule): a broadcast +received from the backbone is relayed on radio only. This cuts the hub fallback from +N×(N−1) to N−1 datagrams per flood, and is the only part that needs a stack change +(gaps S1–S5). + +## Facts checked in the stack + +- Passive-ack audience is every router neighbour with a live link: + `Neighbors::expected_broadcast_relayers()` (`ziggurat-zigbee/src/nwk/neighbors.rs`), + used in `ziggurat-driver/src/zigbee_stack/nwk.rs`. Quorum is + `min(audience, broadcast_passive_ack_quorum = 8)`; below it a broadcast is re-sent + `max_broadcast_retries = 2` times, `passive_ack_timeout = 500 ms` apart. +- A route request is only processed from a neighbour with a non-zero outgoing cost + (`handle_route_request`), so backbone nodes must exchange Link Status to route + through each other. +- Relayed route requests keep the originator's NWK source and sequence number; only + path cost and radius change, and a strictly better cost is relayed again + (`route.rs`). Relays are retried `rreq_retries = 2` times. +- Neighbours age out after `router_age_limit = 3` missed Link Status rounds of + `link_status_period = 15 s`: 45 s. +- Relayed frames are re-secured by each hop (per-hop frame counter and source), so a + relay cannot be forged without the network key. + +## Gap analysis + +Severity: **must** = the design is wrong without it; **minor**; **accepted** = known +limit of the spike; **open** = needs measurement. + +### Multicast + +| ID | Gap | Severity | Resolution | +|---|---|---|---| +| M1 | Nodes listen on different UDP ports (two nodes on one host on the bench); multicast needs one shared port. | must | Second socket on a fixed multicast port with address and port reuse. Multicast is sent from the unicast socket, so receivers learn the sender's unicast address from it. | +| M2 | Link-local multicast is per interface; without `--zloe-interface` the interface is unknown. | must | Multicast only when the backbone interface is known (`--zloe-interface`, or the host has exactly one candidate). Otherwise use the fallback. | +| M3 | IPv4 multicast cannot be joined on the dual-stack socket. | accepted | IPv6 multicast only. IPv4-only links use the fallback. | +| M4 | Multicast can silently stop (snooping switch without a querier, Wi-Fi isolation), and can fail in one direction only. | must | Probe: one multicast datagram per 5 s carries the ack-request flag; peers answer by unicast. A peer that misses two probes is unconfirmed, and the sender adds the fallback. Detection ≤ 10 s, well inside the 45 s neighbour ageing. | +| M5 | Wi-Fi access points buffer multicast for sleeping clients and send it at the lowest rate. Expected +100–300 ms on broadcasts crossing the backbone. | open | Measure with a Wi-Fi satellite. Unicast is unaffected. | +| M6 | Multicast datagrams are unacknowledged. | covered | Each node receives up to N−1 copies of a flood (every relay), and Zigbee re-sends when passive acks are missing. | +| M7 | A node receives its own multicast (loopback must stay on for same-host peers). | covered | Already dropped by the sender-address check. | +| M8 | Any host on the segment can see headers (addresses, extended PAN ID) and inject datagrams. | accepted | Trusted backbone; group derived from the extended PAN ID plus the existing header check; payload stays NWK-encrypted. | +| M9 | Non-NWK broadcasts (beacon requests) are sent over the backbone today. | minor | Only 802.15.4 data frames go to the backbone. | + +### Hub fallback + +| ID | Gap | Severity | Resolution | +|---|---|---|---| +| H1 | Receivers overwrite a peer's IP address with the datagram's UDP source; a forwarded datagram would point every satellite at the coordinator. | must | "Forwarded" flag: receivers keep the sender's stored address and acknowledge the coordinator (hop by hop). The header already carries the original sender (extended and short address). | +| H2 | Satellites that reach the coordinator but not each other (Wi-Fi client isolation, firewall): they hear each other's Link Status through the hub and become neighbours, but direct unicast fails and falls back to radio. | must on such networks | The coordinator also forwards a unicast frame when direct delivery was not acknowledged; the destination is read from the inner 802.15.4 header (no new header field). Link quality reported for the peer follows the path used. | +| H3 | Coordinator down: Link Status between satellites stops, neighbours age out after 45 s and direct routes die. | must | No ack from the coordinator → send to every peer directly (transport 3). | +| H4 | Coordinator just restarted: it acknowledges but does not know every peer yet, so some broadcasts are not forwarded to all, and the sender cannot tell. | accepted | Lasts at most one Link Status round (15 s), under the 45 s ageing. | +| H5 | Without suppression the hub costs N×(N−1) datagrams per flood, as today; route-request relays up to 3× that. | accepted | Paid only where multicast fails. Suppression (below) is the alternative. | +| H6 | Forwarding loops. | covered | A datagram flagged forwarded is never forwarded again; duplicates are dropped by sender and packet number. | +| H7 | Any backbone host can make the coordinator send N−2 datagrams per datagram. | accepted | Trusted backbone. | + +### Suppression (optional) + +| ID | Gap | Severity | Resolution | +|---|---|---|---| +| S1 | Backbone nodes stop hearing each other's relays, so every node misses its passive-ack quorum and re-sends each broadcast twice, on radio too. ZLOE cannot forge the relays. | must | Stack change: leave backbone peers out of `expected_broadcast_relayers()`, with a way for the PHY to say who is a backbone peer. | +| S2 | ZLOE cannot tell a better-cost route request from a duplicate (same NWK source and sequence, cost encrypted). Suppressing it hides better paths, e.g. two satellites hear one device, one well and one badly. | must | Exempt NWK command frames (the frame type is readable): route discovery stays at N×(N−1). Or the stack tells the PHY which link the relayed copy arrived on (larger stack change, optimal). | +| S3 | Each backbone node gets exactly one copy of a flood: no redundancy against loss. | must | Never combine suppression with unacknowledged multicast; hub hops are acked. | +| S4 | Multicast expects passive acks from backbone peers, suppressed hub mode must not; the audience rule would change at runtime per peer. | must | Needs a per-peer transport state visible to the stack, or suppression in both modes (which conflicts with S3). | +| S5 | The "arrived from backbone" table needs a lifetime of at least `broadcast_delivery_time` (9 s) and must survive 8-bit sequence wrap. | minor | Timed entries keyed by NWK source and sequence. | + +### Result + +- Multicast and the hub fallback need **no stack change** and add no new Zigbee + behaviour. +- Every hard gap (S1–S4) comes from suppression, which only improves the fallback + path, and only on networks where multicast does not work. +- Without suppression the hub's total datagram count equals today's direct fan-out. + What the hub adds is one datagram per broadcast for the satellite instead of N−1 + (matters for microcontroller or Wi-Fi satellites) and reachability between isolated + satellites (H2). + +Recommended order: (1) multicast + probe, with today's direct fan-out as the fallback; +(2) hub forwarding with H1–H3; (3) suppression only if measurements on a +multicast-hostile network show the fallback traffic is a problem. + +## Wire changes + +- Flags: bit 0 ack request (exists; on a multicast datagram it is the probe), bit 1 + "forward this" (satellite → coordinator), bit 2 "forwarded" (coordinator → peers). +- Multicast group: `ff02::5a4c:hhhh:hhhh`, `h` = low 32 bits of the extended PAN ID. +- Multicast port: fixed, configurable (`--zloe-mcast-port`). +- No new header fields: the original sender is already in the header, a forwarded + unicast's destination is in the inner 802.15.4 header. + +## Behaviour + +``` +Backbone (ziggurat-phy-trel) + send_broadcast + Every known peer has acknowledged a recent multicast probe + - sends one datagram to the network's multicast group + - sends no unicast copies + A peer has missed two probes, node is a satellite + - sends the multicast datagram + - sends one acked datagram to the coordinator with the forward flag and the same packet number + A peer has missed two probes, node is the coordinator + - sends the multicast datagram + - sends one acked datagram to each unconfirmed peer with the same packet number + Multicast unavailable (no known interface, IPv4-only link) + - satellite sends one acked datagram to the coordinator with the forward flag + - coordinator sends one acked datagram to each peer + Coordinator does not acknowledge + - sends one acked datagram to every known peer + - lowers the coordinator's backbone link quality + Frame is not an 802.15.4 data frame + - sends nothing over the backbone + Node has no network identity yet + - sends nothing over the backbone + receive_multicast + Datagram from another node on our extended PAN ID + - delivers the frame to the stack once, 802.15.4 source unchanged + - stores the datagram's source as the sender's unicast address + - sends a unicast ack when the ack-request flag is set + Datagram sent by this node (loopback) + - drops it + Packet number already received from that sender + - drops it + Different extended PAN ID + - drops it + forward (coordinator) + Broadcast with the forward flag from a satellite + - acknowledges the satellite + - delivers the frame to the local stack once + - sends it unchanged with the forwarded flag to every other peer, acked per hop + Same sender and packet number seen before + - acknowledges, forwards nothing + Datagram already flagged forwarded + - does not forward it + Unicast with the forward flag whose 802.15.4 destination is a known peer + - sends it with the forwarded flag to that peer only + Unicast with the forward flag for an unknown destination + - acknowledges, forwards nothing + receive_forwarded + Datagram flagged forwarded + - delivers the frame to the stack once, 802.15.4 source unchanged + - leaves the original sender's stored address unchanged + - acknowledges the coordinator + send_unicast + Peer acknowledges directly + - returns Acked, sends nothing to the coordinator + Peer does not acknowledge directly, node is a satellite + - sends the frame to the coordinator with the forward flag + - returns Acked when the coordinator acknowledges + Neither path acknowledges + - falls back to the radio + probe + Probe interval elapsed + - sets the ack-request flag on the next multicast datagram + - sends an empty probe when no broadcast was due within the interval + Peer acknowledges a probe + - marks multicast to that peer confirmed + Peer misses two consecutive probes + - marks multicast to that peer unconfirmed + - logs the transition once + +Optional suppression + Backbone + send_broadcast + NWK data broadcast that arrived from the backbone within the delivery time + - transmits on radio only + NWK command broadcast + - is sent to the backbone as any other broadcast + Neighbors + expected_broadcast_relayers + Neighbour is a backbone peer + - is not part of the passive-ack audience + Neighbour is radio-only + - is part of the audience as today +``` + +## Verification on the bench + +- Count datagrams per flooded broadcast (device announce from a ZBOSS router) in each + transport with 2 and 3 backbone nodes. +- Block multicast between the two bench nodes and check the switch to the fallback + within 10 s and back after unblocking. +- Stop the coordinator for 60 s with two satellites and check that they stay + neighbours. +- Re-run the deny experiment (On/Off through the satellite) in each transport. From 0990a867d40e758b7be09ce0aaf1f1a5f88c670e Mon Sep 17 00:00:00 2001 From: Igor Yamolov Date: Tue, 10 Nov 2026 15:40:06 +0100 Subject: [PATCH 10/22] SPIKE: announce the ZLOE multicast address over mDNS - Join the network's group ff12::5a4c: on the backbone interface (--zloe-mcast-port, default 19789, needs --zloe-interface) and announce it as TXT mc=[group]:port; a node that cannot receive multicast announces none - Store each peer's announced address, show it in [diag] - Keep mDNS results received before the node has an identity and apply them once it has one (a satellite missed the coordinator's record) - Datagrams arriving on the group go through the normal receive path; sending broadcasts by multicast is not built yet Co-Authored-By: Claude --- crates/ziggurat-phy-trel/BROADCAST.md | 51 ++++- crates/ziggurat-phy-trel/Cargo.toml | 2 +- crates/ziggurat-phy-trel/src/lib.rs | 212 +++++++++++++++++---- crates/ziggurat-phy-trel/tests/loopback.rs | 1 + crates/ziggurat-server/src/main.rs | 6 + 5 files changed, 230 insertions(+), 42 deletions(-) diff --git a/crates/ziggurat-phy-trel/BROADCAST.md b/crates/ziggurat-phy-trel/BROADCAST.md index aab5750..eefc10b 100644 --- a/crates/ziggurat-phy-trel/BROADCAST.md +++ b/crates/ziggurat-phy-trel/BROADCAST.md @@ -1,6 +1,8 @@ # ZLOE broadcast delivery: design and gap analysis -Status: design for the spike (zigpy/ziggurat#84). Nothing here is built yet. +Status: design for the spike (zigpy/ziggurat#84). Built so far: the multicast +announcement and listener (see "Multicast announcement"). Sending broadcasts by +multicast, the probe and the hub are not built. Styleguide: the repository has no written styleguide; its conventions are enforced by `.pre-commit-config.yaml` (`cargo fmt --all`, `cargo clippy --all-targets @@ -67,8 +69,8 @@ limit of the spike; **open** = needs measurement. | ID | Gap | Severity | Resolution | |---|---|---|---| -| M1 | Nodes listen on different UDP ports (two nodes on one host on the bench); multicast needs one shared port. | must | Second socket on a fixed multicast port with address and port reuse. Multicast is sent from the unicast socket, so receivers learn the sender's unicast address from it. | -| M2 | Link-local multicast is per interface; without `--zloe-interface` the interface is unknown. | must | Multicast only when the backbone interface is known (`--zloe-interface`, or the host has exactly one candidate). Otherwise use the fallback. | +| M1 | Nodes listen on different UDP ports (two nodes on one host on the bench); multicast needs one shared port. | must | Second socket on the multicast port with address and port reuse (built). Each node announces its group and port over mDNS; a sender uses one datagram per distinct announced address, normally one. Multicast is sent from the unicast socket, so receivers learn the sender's unicast address from it. | +| M2 | Link-local multicast is per interface; without `--zloe-interface` the interface is unknown. | must | Multicast only with `--zloe-interface` (built). A node without it announces no multicast address, so peers reach it through the fallback. | | M3 | IPv4 multicast cannot be joined on the dual-stack socket. | accepted | IPv6 multicast only. IPv4-only links use the fallback. | | M4 | Multicast can silently stop (snooping switch without a querier, Wi-Fi isolation), and can fail in one direction only. | must | Probe: one multicast datagram per 5 s carries the ack-request flag; peers answer by unicast. A peer that misses two probes is unconfirmed, and the sender adds the fallback. Detection ≤ 10 s, well inside the 45 s neighbour ageing. | | M5 | Wi-Fi access points buffer multicast for sleeping clients and send it at the lowest rate. Expected +100–300 ms on broadcasts crossing the backbone. | open | Measure with a Wi-Fi satellite. Unicast is unaffected. | @@ -118,11 +120,31 @@ multicast-hostile network show the fallback traffic is a problem. - Flags: bit 0 ack request (exists; on a multicast datagram it is the probe), bit 1 "forward this" (satellite → coordinator), bit 2 "forwarded" (coordinator → peers). -- Multicast group: `ff02::5a4c:hhhh:hhhh`, `h` = low 32 bits of the extended PAN ID. -- Multicast port: fixed, configurable (`--zloe-mcast-port`). +- Multicast group: `ff12::5a4c:hhhh:hhhh` (transient, link-local scope), `h` = low 32 + bits of the extended PAN ID. +- Multicast port: `--zloe-mcast-port`, default 19789, 0 disables multicast. +- mDNS TXT `mc=[group]:port`: where the node listens for multicast. - No new header fields: the original sender is already in the header, a forwarded unicast's destination is in the inner 802.15.4 header. +## Multicast announcement + +Built. A node that can receive multicast joins the network's group on the backbone +interface and announces `mc=[ff12::5a4c:hhhh:hhhh]:port` in its `_zloe._udp` TXT +record. A node that cannot (no `--zloe-interface`, no IPv6, port 0) announces no `mc`. + +- The announcement says a peer *can* receive multicast; the probe (M4) says whether + it *does*. A peer without `mc` is never counted as reachable by multicast. +- Peers store the announced address scoped to their own backbone interface. It is + cleared when the peer's mDNS record goes away. +- mDNS results are now kept while a node has no identity and applied once it has one: + a satellite browses before it knows its network, and used to miss everything the + coordinator announced. + +Bench check: both nodes joined `ff12::5a4c:f5d9:7c12` on `enp4s0f0` (kernel shows two +users, two sockets on port 19789), each lists the other's `mc`, and a test datagram +sent to the group reached both. + ## Behaviour ``` @@ -147,6 +169,25 @@ Backbone (ziggurat-phy-trel) - sends nothing over the backbone Node has no network identity yet - sends nothing over the backbone + announce + Backbone interface known, IPv6 available, multicast port set + - joins the network's group on that interface + - announces TXT mc=[group]:port + Interface unknown, no IPv6, or multicast port 0 + - joins no group + - announces no mc + Network (extended PAN ID) changes + - leaves the old group and joins the new one + - announces the new mc + resolve_peer + Peer announces a multicast address + - stores it scoped to our backbone interface + Peer announces none, or an address that is not multicast + - stores none; the peer is reached by unicast only + Peer's record is removed + - clears the stored multicast address + Record resolved before this node has an identity + - is applied when the identity becomes known receive_multicast Datagram from another node on our extended PAN ID - delivers the frame to the stack once, 802.15.4 source unchanged diff --git a/crates/ziggurat-phy-trel/Cargo.toml b/crates/ziggurat-phy-trel/Cargo.toml index 06e5bdf..18d7adc 100644 --- a/crates/ziggurat-phy-trel/Cargo.toml +++ b/crates/ziggurat-phy-trel/Cargo.toml @@ -17,7 +17,7 @@ tokio = { version = "1.53.1", features = ["rt", "time", "sync", "net", "macros"] tracing = "0.1" mdns-sd = "0.21.7" if-addrs = "0.15" -socket2 = "0.6.5" +socket2 = { version = "0.6.5", features = ["all"] } [dev-dependencies] tokio = { version = "1.53.1", features = ["rt-multi-thread", "macros", "time", "sync", "net"] } diff --git a/crates/ziggurat-phy-trel/src/lib.rs b/crates/ziggurat-phy-trel/src/lib.rs index b3b486e..2aa8381 100644 --- a/crates/ziggurat-phy-trel/src/lib.rs +++ b/crates/ziggurat-phy-trel/src/lib.rs @@ -90,6 +90,9 @@ pub struct TrelConfig { pub interface: Option, /// Reach peers over IPv4 when they advertise both (default: IPv6 link-local) pub prefer_ipv4: bool, + /// UDP port of the network's multicast group (0: do not listen or advertise). + /// Needs `interface`: link-local multicast is per interface. + pub mcast_port: u16, pub deny: DenyList, } @@ -134,6 +137,9 @@ struct Peer { health: f32, /// Every usable address mDNS gave, best first: tried in turn when sends fail alternates: Vec, + /// The multicast group and port the peer listens on (mDNS TXT `mc`), scoped to + /// our backbone interface. `None`: the peer cannot be reached by multicast. + mcast: Option, } impl Peer { @@ -145,6 +151,7 @@ impl Peer { last_rx: None, health: 1.0, alternates: Vec::new(), + mcast: None, } } @@ -206,6 +213,8 @@ struct Shared { /// Coordinators seen on the backbone (mDNS `role=coordinator`), kept even before /// we have an identity of our own: satellites bootstrap from them coordinators: HashMap)>, + /// Everything mDNS resolved, by extended address + resolved: HashMap, netinfo_provider: Option, netinfo_waiter: Option>>, } @@ -223,6 +232,9 @@ struct Backbone { prefer_ipv4: bool, /// The socket is IPv6 dual-stack: IPv4 targets go out as v4-mapped addresses dual_stack: bool, + mcast_port: u16, + /// The task reading our multicast socket; replaced when the network changes + mcast_reader: Mutex>>, } pub struct TrelPhy { @@ -303,8 +315,10 @@ impl TrelPhy

{ local_net, prefer_ipv4: config.prefer_ipv4, dual_stack, + mcast_port: config.mcast_port, + mcast_reader: Mutex::new(None), }; - backbone.spawn_udp_reader(); + backbone.spawn_udp_reader(Arc::clone(&backbone.socket)); backbone.spawn_mdns_browser(); Some(backbone) }; @@ -407,6 +421,58 @@ fn netinfo_header(kind: u8, seq: u32, identity: Option) -> Vec { .to_bytes() } +/// What mDNS last told us about a node. +#[derive(Debug, Clone)] +struct Resolved { + epan: Option, + nwk: Option, + /// Usable addresses, best first, with their rank + candidates: Vec<((u8, u8), SocketAddr)>, + mcast: Option, +} + +/// Create or update the peer for a resolved node on our network. +fn apply_resolved(shared: &mut Shared, eui64: Eui64) { + let Some(identity) = shared.identity else { + return; + }; + let Some(resolved) = shared.resolved.get(&eui64).cloned() else { + return; + }; + if eui64 == identity.eui64 || resolved.epan != Some(identity.epan) { + return; + } + let Some(&(best_key, addr)) = resolved.candidates.first() else { + return; + }; + let nwk = resolved.nwk; + + let peer = shared.peers.entry(eui64).or_insert_with(|| { + tracing::info!("ZLOE: mDNS peer {eui64:?} / {nwk:?} at {addr}"); + Peer::new(addr, nwk, PeerSource::Mdns) + }); + // Once the peer has talked to us, its datagrams' source address is authoritative. + // Until then follow mDNS, but only to a strictly better address: answers trickle + // in per interface and would make it flap + let current_key = resolved + .candidates + .iter() + .find(|(_, a)| *a == peer.addr) + .map(|(k, _)| *k); + peer.alternates = resolved.candidates.iter().map(|(_, a)| *a).collect(); + if peer.mcast != resolved.mcast { + tracing::info!("ZLOE: peer {eui64:?} multicast: {:?}", resolved.mcast); + peer.mcast = resolved.mcast; + } + if peer.last_rx.is_none() && current_key.is_none_or(|k| best_key < k) { + tracing::info!("ZLOE: mDNS peer {eui64:?} moved to {addr}"); + peer.addr = addr; + } + if nwk.is_some() { + peer.nwk = nwk; + } +} + /// Combine the radio and network halves into the identity, once both are known. fn update_identity(shared: &mut Shared) -> Option { let (eui64, nwk, channel) = shared.radio?; @@ -421,6 +487,13 @@ fn update_identity(shared: &mut Shared) -> Option { old.eui64 != identity.eui64 || old.nwk != identity.nwk || old.epan != identity.epan }); shared.identity = Some(identity); + if changed { + // Nodes resolved before we knew our network become peers now + let known: Vec = shared.resolved.keys().copied().collect(); + for eui64 in known { + apply_resolved(shared, eui64); + } + } changed.then_some(identity) } @@ -458,7 +531,9 @@ fn canonical(addr: SocketAddr) -> SocketAddr { } impl Backbone { - fn spawn_udp_reader(&self) { + /// Read datagrams from `rx` (the unicast socket, or the multicast one). Acks and + /// replies always leave from the unicast socket, so peers see our unicast port. + fn spawn_udp_reader(&self, rx: Arc) -> tokio::task::JoinHandle<()> { let socket = Arc::clone(&self.socket); let shared = Arc::clone(&self.shared); let pending = Arc::clone(&self.pending); @@ -467,7 +542,7 @@ impl Backbone { tokio::spawn(async move { let mut buf = vec![0u8; 2048]; loop { - let (len, from) = match socket.recv_from(&mut buf).await { + let (len, from) = match rx.recv_from(&mut buf).await { Ok((len, from)) => (len, canonical(from)), Err(e) => { tracing::warn!("ZLOE: UDP receive failed: {e}"); @@ -584,7 +659,7 @@ impl Backbone { let _ = tx.try_send(frame); } } - }); + }) } fn spawn_mdns_browser(&self) { @@ -616,6 +691,16 @@ impl Backbone { let nwk = info .get_property_val_str("nwk") .and_then(|s| Nwk::try_from_hex(s).ok()); + // Usable only on a known interface: the group is link-local + let mcast = info + .get_property_val_str("mc") + .and_then(|s| s.parse::().ok()) + .filter(|mc| mc.ip().is_multicast()) + .zip(local_net) + .map(|(mut mc, net)| { + mc.set_scope_id(net.index); + mc + }); // mDNS lists every address of every interface (Docker bridges, // loopback, ...): pick deterministically let candidates = candidate_addresses( @@ -624,11 +709,9 @@ impl Backbone { local_net, prefer_ipv4, ); - let Some(&(best_key, addr)) = candidates.first() else { + let Some(&(_, addr)) = candidates.first() else { continue; }; - let alternates: Vec = - candidates.iter().map(|(_, a)| *a).collect(); let mut shared = shared.lock(); if info.get_property_val_str("role") == Some("coordinator") { @@ -639,33 +722,18 @@ impl Backbone { ); } } - let Some(identity) = shared.identity else { - continue; - }; - if eui64 == identity.eui64 || epan != Some(identity.epan) { - continue; - } - let peer = shared.peers.entry(eui64).or_insert_with(|| { - tracing::info!("ZLOE: mDNS peer {eui64:?} / {nwk:?} at {addr}"); - Peer::new(addr, nwk, PeerSource::Mdns) - }); - // Once the peer has talked to us, its datagrams' source address - // is authoritative - // Until the peer talks to us, follow mDNS, but only to a - // strictly better address: answers trickle in per interface - // and would make it flap - let current_key = candidates - .iter() - .find(|(_, a)| *a == peer.addr) - .map(|(k, _)| *k); - peer.alternates = alternates; - if peer.last_rx.is_none() && current_key.is_none_or(|k| best_key < k) { - tracing::info!("ZLOE: mDNS peer {eui64:?} moved to {addr}"); - peer.addr = addr; - } - if nwk.is_some() { - peer.nwk = nwk; - } + // Kept even while we have no identity (a satellite browses + // before it knows its network) and applied once we do + shared.resolved.insert( + eui64, + Resolved { + epan, + nwk, + candidates, + mcast, + }, + ); + apply_resolved(&mut shared, eui64); } mdns_sd::ServiceEvent::ServiceRemoved(_, fullname) => { let Some(eui64) = fullname @@ -676,6 +744,12 @@ impl Backbone { continue; }; let mut shared = shared.lock(); + shared.resolved.remove(&eui64); + if let Some(peer) = shared.peers.get_mut(&eui64) { + // A peer kept alive by its own traffic loses what only + // mDNS told us + peer.mcast = None; + } if shared .peers .get(&eui64) @@ -691,7 +765,50 @@ impl Backbone { }); } + /// Join the network's multicast group and read from it. `None` when multicast is + /// off, the backbone interface is unknown, or the host has no IPv6. + fn listen_multicast(&self, epan: Eui64) -> Option { + use socket2::{Domain, Protocol, Socket, Type}; + + if let Some(old) = self.mcast_reader.lock().take() { + old.abort(); // closes the socket, leaving the previous group + } + if self.mcast_port == 0 || !self.dual_stack { + return None; + } + let Some(net) = self.local_net else { + tracing::info!("ZLOE: no multicast without --zloe-interface"); + return None; + }; + + let group = multicast_group(epan); + let open = || -> std::io::Result { + let socket = Socket::new(Domain::IPV6, Type::DGRAM, Some(Protocol::UDP))?; + socket.set_only_v6(true)?; + // Several nodes on one host (the bench) share the port + socket.set_reuse_address(true)?; + socket.set_reuse_port(true)?; + socket.bind(&SocketAddr::from((Ipv6Addr::UNSPECIFIED, self.mcast_port)).into())?; + socket.join_multicast_v6(&group, net.index)?; + socket.set_nonblocking(true)?; + UdpSocket::from_std(socket.into()) + }; + match open() { + Ok(socket) => { + let mc = SocketAddrV6::new(group, self.mcast_port, 0, net.index); + tracing::info!("ZLOE: listening on multicast group {mc}"); + *self.mcast_reader.lock() = Some(self.spawn_udp_reader(Arc::new(socket))); + Some(mc) + } + Err(e) => { + tracing::warn!("ZLOE: cannot join multicast group {group}: {e}"); + None + } + } + } + fn advertise(&self, identity: Identity) { + let mcast = self.listen_multicast(identity.epan); let Some(daemon) = &self.mdns else { return }; let mut shared = self.shared.lock(); if let Some(old) = shared.mdns_fullname.take() { @@ -699,7 +816,7 @@ impl Backbone { } let instance = hex_be(&identity.eui64.to_be_bytes()); - let properties: HashMap = [ + let mut properties: HashMap = [ ("v".to_string(), VERSION.to_string()), ("xa".to_string(), instance.clone()), ("xp".to_string(), hex_be(&identity.epan.to_be_bytes())), @@ -717,6 +834,12 @@ impl Backbone { .into_iter() .collect(); + // `mc`: where we listen for multicast. Absent when we cannot, so peers know + // to reach us by unicast only + if let Some(mc) = mcast { + properties.insert("mc".to_string(), format!("[{}]:{}", mc.ip(), mc.port())); + } + // Addresses (IPv6 link-local and IPv4) come from the interfaces mDNS runs on, // which --zloe-interface already narrows to one let host = format!("ziggurat-{instance}.local."); @@ -981,11 +1104,12 @@ impl TrelPhy

{ .iter() .map(|(eui64, p)| { format!( - "{eui64:?} nwk={:?} addr={} via={:?} lqi={} last_rx={:?}", + "{eui64:?} nwk={:?} addr={} via={:?} lqi={} mc={} last_rx={:?}", p.nwk, p.addr, p.source, p.lqi(), + p.mcast.map_or_else(|| "-".to_string(), |mc| mc.to_string()), p.last_rx.map(|t| t.elapsed()) ) }) @@ -1156,6 +1280,22 @@ fn candidate_addresses( ranked } +/// The network's multicast group: transient (`ff1x`), link-local scope (`ffx2`), with +/// the low 32 bits of the extended PAN ID: `ff12::5a4c:hhhh:hhhh`. +fn multicast_group(epan: Eui64) -> Ipv6Addr { + let e = epan.to_be_bytes(); + Ipv6Addr::new( + 0xff12, + 0, + 0, + 0, + 0, + 0x5a4c, + u16::from_be_bytes([e[4], e[5]]), + u16::from_be_bytes([e[6], e[7]]), + ) +} + /// IPv4 targets on a dual-stack IPv6 socket must be v4-mapped. fn to_family(addr: SocketAddr, dual_stack: bool) -> SocketAddr { match addr { diff --git a/crates/ziggurat-phy-trel/tests/loopback.rs b/crates/ziggurat-phy-trel/tests/loopback.rs index 7612c92..0128f9d 100644 --- a/crates/ziggurat-phy-trel/tests/loopback.rs +++ b/crates/ziggurat-phy-trel/tests/loopback.rs @@ -119,6 +119,7 @@ async fn node( mdns, interface: None, prefer_ipv4: false, + mcast_port: 0, deny: DenyList::default(), }, ) diff --git a/crates/ziggurat-server/src/main.rs b/crates/ziggurat-server/src/main.rs index 8286af0..31d08aa 100644 --- a/crates/ziggurat-server/src/main.rs +++ b/crates/ziggurat-server/src/main.rs @@ -1047,6 +1047,11 @@ struct Args { #[arg(long)] zloe_ipv4: bool, + /// SPIKE: UDP port of the ZLOE multicast group, announced over mDNS (TXT `mc`); + /// needs --zloe-interface. 0 disables multicast + #[arg(long, default_value_t = 19789)] + zloe_mcast_port: u16, + /// SPIKE: disable mDNS discovery of ZLOE peers #[arg(long, alias = "no-mdns")] zloe_no_mdns: bool, @@ -1140,6 +1145,7 @@ fn main() -> Result<(), Box> { mdns: !args.zloe_no_mdns, interface: args.zloe_interface, prefer_ipv4: args.zloe_ipv4, + mcast_port: args.zloe_mcast_port, deny, }, )); From 127bdcd18c45f17ee836bf0158be6c4e378d1dfc Mon Sep 17 00:00:00 2001 From: Igor Yamolov Date: Tue, 10 Nov 2026 15:52:31 +0100 Subject: [PATCH 11/22] SPIKE: send ZLOE broadcasts by multicast with probe and fallback - Broadcasts go out as one multicast datagram to the peers whose multicast is confirmed, and as a unicast copy (same packet number) to every other peer - Probe the group every 5 s; peers answer by unicast; a peer without an ack for 12 s gets unicast copies again - Only 802.15.4 data frames go to the backbone - [diag] datagram counters and a test hook that drops group datagrams while a file exists - Bench: fallback after 15 s of blocked multicast, recovery 10 s after unblocking, the backbone neighbour link held throughout Co-Authored-By: Claude Opus 5.5 --- crates/ziggurat-phy-trel/BROADCAST.md | 31 ++- crates/ziggurat-phy-trel/src/lib.rs | 220 +++++++++++++++++++-- crates/ziggurat-phy-trel/tests/loopback.rs | 1 + crates/ziggurat-server/src/main.rs | 9 + 4 files changed, 238 insertions(+), 23 deletions(-) diff --git a/crates/ziggurat-phy-trel/BROADCAST.md b/crates/ziggurat-phy-trel/BROADCAST.md index eefc10b..644983c 100644 --- a/crates/ziggurat-phy-trel/BROADCAST.md +++ b/crates/ziggurat-phy-trel/BROADCAST.md @@ -1,8 +1,8 @@ # ZLOE broadcast delivery: design and gap analysis -Status: design for the spike (zigpy/ziggurat#84). Built so far: the multicast -announcement and listener (see "Multicast announcement"). Sending broadcasts by -multicast, the probe and the hub are not built. +Status: design for the spike (zigpy/ziggurat#84). Step 1 is built: multicast +announcement, broadcasts by multicast, the probe, and per-peer unicast copies as the +fallback (see "Step 1 as built"). The hub and suppression are not built. Styleguide: the repository has no written styleguide; its conventions are enforced by `.pre-commit-config.yaml` (`cargo fmt --all`, `cargo clippy --all-targets @@ -145,6 +145,31 @@ Bench check: both nodes joined `ff12::5a4c:f5d9:7c12` on `enp4s0f0` (kernel show users, two sockets on port 19789), each lists the other's `mc`, and a test datagram sent to the group reached both. +## Step 1 as built + +- A broadcast goes out as one multicast datagram to each distinct group of peers whose + multicast is confirmed, plus a unicast copy (same packet number) to every other peer. + Receivers keep one copy per sender and packet number. +- Only 802.15.4 data frames go to the backbone; beacon requests stay on the radio (M9). +- Probe: every 5 s an empty `KIND_PROBE` datagram (kind 4) is multicast with the + ack-request flag; each peer answers with a unicast ack. A peer is confirmed while it + has acked a probe in the last 12 s. A peer that announces no `mc` is never confirmed. +- `[diag] backbone broadcasts=… multicast=… unicast_copies=… probes=…` counts + datagrams since start. +- Test hook `--zloe-test-mcast-block-file `: while the file exists, datagrams + arriving on the group are dropped. + +Bench (coordinator and satellite, `enp4s0f0`, IPv6 link-local): + +- Both sides confirmed multicast within 2 s of meeting; broadcasts went out as one + multicast datagram with no unicast copies. +- Satellite multicast blocked at 15:51:08: the coordinator switched it to unicast + copies at 15:51:23 (15 s). Unblocked at 15:51:48: back to multicast at 15:51:58 + (10 s). The satellite stayed a cost-1 neighbour throughout. +- Detection takes 12–17 s (12 s window plus up to one 5 s probe tick), not the + ≤ 10 s M4 aimed at; still well inside the 45 s neighbour ageing. Broadcasts sent in + that gap reach the peer only through Zigbee's own retries. + ## Behaviour ``` diff --git a/crates/ziggurat-phy-trel/src/lib.rs b/crates/ziggurat-phy-trel/src/lib.rs index 2aa8381..e760aae 100644 --- a/crates/ziggurat-phy-trel/src/lib.rs +++ b/crates/ziggurat-phy-trel/src/lib.rs @@ -27,7 +27,7 @@ use std::collections::{HashMap, HashSet, VecDeque}; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, SocketAddrV6}; use std::sync::Arc; -use std::sync::atomic::{AtomicU32, Ordering}; +use std::sync::atomic::{AtomicU32, AtomicU64, Ordering}; use std::time::{Duration, Instant}; use parking_lot::Mutex; @@ -50,6 +50,14 @@ const KIND_NETINFO_RSP: u8 = 3; /// Produces the encoded network settings a coordinator hands to satellites. pub type NetInfoProvider = Arc Option> + Send + Sync>; const FLAG_ACK_REQUEST: u8 = 0x01; +/// Multicast liveness probe: no payload, answered with a unicast ack. +const KIND_PROBE: u8 = 4; +const PROBE_INTERVAL: Duration = Duration::from_secs(5); +/// A peer that has not acked a probe for this long (two missed probes) gets unicast +/// copies of broadcasts again. +const MCAST_CONFIRM_WINDOW: Duration = Duration::from_secs(12); +/// 802.15.4 frame type "data": the only broadcasts that belong on the backbone. +const MAC_FRAME_DATA: u8 = 1; const ACK_TIMEOUT: Duration = Duration::from_millis(50); const ATTEMPTS: usize = 3; @@ -93,6 +101,9 @@ pub struct TrelConfig { /// UDP port of the network's multicast group (0: do not listen or advertise). /// Needs `interface`: link-local multicast is per interface. pub mcast_port: u16, + /// Test hook: while this file exists, datagrams arriving on the multicast group + /// are dropped, as if the network had stopped delivering multicast to us. + pub mcast_block_file: Option, pub deny: DenyList, } @@ -140,6 +151,10 @@ struct Peer { /// The multicast group and port the peer listens on (mDNS TXT `mc`), scoped to /// our backbone interface. `None`: the peer cannot be reached by multicast. mcast: Option, + /// When the peer last acked one of our multicast probes + mcast_acked: Option, + /// The confirmation state last logged, to log each change once + mcast_logged_ok: bool, } impl Peer { @@ -152,9 +167,20 @@ impl Peer { health: 1.0, alternates: Vec::new(), mcast: None, + mcast_acked: None, + mcast_logged_ok: false, } } + /// The peer announces a multicast address and has acked a recent probe to it: our + /// multicast reaches it. + fn multicast_ok(&self) -> bool { + self.mcast.is_some() + && self + .mcast_acked + .is_some_and(|t| t.elapsed() < MCAST_CONFIRM_WINDOW) + } + /// The current address went unanswered: move to the next advertised one. fn rotate(&mut self) { if self.alternates.len() < 2 { @@ -217,13 +243,29 @@ struct Shared { resolved: HashMap, netinfo_provider: Option, netinfo_waiter: Option>>, + /// Packet numbers of our latest multicast probes + probes: VecDeque, +} + +/// Datagram counters, for the bench. +#[derive(Default)] +struct Stats { + /// 802.15.4 broadcasts handed to the backbone + broadcasts: AtomicU64, + /// Multicast datagrams they were sent as + multicast: AtomicU64, + /// Unicast copies sent to peers multicast does not reach + unicast_copies: AtomicU64, + probes: AtomicU64, } struct Backbone { socket: Arc, shared: Arc>, pending: Arc>>>, - seq: AtomicU32, + seq: Arc, + stats: Arc, + mcast_block_file: Option, static_peers: Vec, mdns: Option, port: u16, @@ -308,7 +350,9 @@ impl TrelPhy

{ socket, shared: Arc::clone(&shared), pending: Arc::new(Mutex::new(HashMap::new())), - seq: AtomicU32::new(rand_u32()), + seq: Arc::new(AtomicU32::new(rand_u32())), + stats: Arc::new(Stats::default()), + mcast_block_file: config.mcast_block_file, static_peers: config.static_peers, mdns, port: config.port, @@ -318,8 +362,9 @@ impl TrelPhy

{ mcast_port: config.mcast_port, mcast_reader: Mutex::new(None), }; - backbone.spawn_udp_reader(Arc::clone(&backbone.socket)); + backbone.spawn_udp_reader(Arc::clone(&backbone.socket), None); backbone.spawn_mdns_browser(); + backbone.spawn_prober(); Some(backbone) }; @@ -463,6 +508,7 @@ fn apply_resolved(shared: &mut Shared, eui64: Eui64) { if peer.mcast != resolved.mcast { tracing::info!("ZLOE: peer {eui64:?} multicast: {:?}", resolved.mcast); peer.mcast = resolved.mcast; + peer.mcast_acked = None; } if peer.last_rx.is_none() && current_key.is_none_or(|k| best_key < k) { tracing::info!("ZLOE: mDNS peer {eui64:?} moved to {addr}"); @@ -533,7 +579,11 @@ fn canonical(addr: SocketAddr) -> SocketAddr { impl Backbone { /// Read datagrams from `rx` (the unicast socket, or the multicast one). Acks and /// replies always leave from the unicast socket, so peers see our unicast port. - fn spawn_udp_reader(&self, rx: Arc) -> tokio::task::JoinHandle<()> { + fn spawn_udp_reader( + &self, + rx: Arc, + block_file: Option, + ) -> tokio::task::JoinHandle<()> { let socket = Arc::clone(&self.socket); let shared = Arc::clone(&self.shared); let pending = Arc::clone(&self.pending); @@ -550,14 +600,26 @@ impl Backbone { continue; } }; + if block_file.as_deref().is_some_and(std::path::Path::exists) { + continue; + } let Some(header) = WireHeader::parse(&buf[..len]) else { tracing::debug!("ZLOE: dropping malformed datagram from {from}"); continue; }; if header.kind == KIND_ACK { - if let Some(waiter) = pending.lock().remove(&header.seq) { + let waiter = pending.lock().remove(&header.seq); + if let Some(waiter) = waiter { let _ = waiter.send(()); + } else { + // Not a unicast we are waiting on: an answer to a probe? + let mut shared = shared.lock(); + if shared.probes.contains(&header.seq) + && let Some(peer) = shared.peers.get_mut(&header.eui64) + { + peer.mcast_acked = Some(Instant::now()); + } } continue; } @@ -588,6 +650,9 @@ impl Backbone { continue; } + if header.kind != KIND_DATA && header.kind != KIND_PROBE { + continue; + } let psdu = buf[HEADER_LEN..len].to_vec(); let (deliver, ack_to) = { let mut shared = shared.lock(); @@ -612,8 +677,9 @@ impl Backbone { peer.last_rx = Some(Instant::now()); let lqi = peer.lqi(); + // A probe only refreshes the peer and gets its ack let seen = shared.seen.entry(header.eui64).or_default(); - let duplicate = seen.contains(&header.seq); + let duplicate = header.kind == KIND_PROBE || seen.contains(&header.seq); if !duplicate { seen.push_back(header.seq); if seen.len() > DEDUP_WINDOW { @@ -797,7 +863,8 @@ impl Backbone { Ok(socket) => { let mc = SocketAddrV6::new(group, self.mcast_port, 0, net.index); tracing::info!("ZLOE: listening on multicast group {mc}"); - *self.mcast_reader.lock() = Some(self.spawn_udp_reader(Arc::new(socket))); + *self.mcast_reader.lock() = + Some(self.spawn_udp_reader(Arc::new(socket), self.mcast_block_file.clone())); Some(mc) } Err(e) => { @@ -859,16 +926,69 @@ impl Backbone { } } - /// Every address a broadcast goes to: known peers plus unidentified static peers. - fn broadcast_targets(&self) -> Vec { - let shared = self.shared.lock(); - let mut targets: Vec = shared.peers.values().map(|p| p.addr).collect(); - for addr in &self.static_peers { - if !targets.contains(addr) { - targets.push(*addr); + /// Every probe interval: multicast an empty datagram that each peer answers by + /// unicast, so we know which peers our multicast reaches. + fn spawn_prober(&self) { + let socket = Arc::clone(&self.socket); + let shared = Arc::clone(&self.shared); + let seq = Arc::clone(&self.seq); + let stats = Arc::clone(&self.stats); + + tokio::spawn(async move { + let mut tick = tokio::time::interval(PROBE_INTERVAL); + loop { + tick.tick().await; + let (probe, groups) = { + let mut shared = shared.lock(); + let Some(identity) = shared.identity else { + continue; + }; + + let mut groups: Vec = Vec::new(); + for (eui64, peer) in &mut shared.peers { + let Some(group) = peer.mcast else { continue }; + if !groups.contains(&group) { + groups.push(group); + } + let ok = peer.multicast_ok(); + if ok != peer.mcast_logged_ok { + peer.mcast_logged_ok = ok; + if ok { + tracing::info!("ZLOE: multicast reaches {eui64:?}"); + } else { + tracing::warn!( + "ZLOE: multicast does not reach {eui64:?}, \ + sending it unicast copies" + ); + } + } + } + if groups.is_empty() { + continue; + } + + let seq = seq.fetch_add(1, Ordering::Relaxed); + shared.probes.push_back(seq); + if shared.probes.len() > 4 { + shared.probes.pop_front(); + } + let probe = WireHeader { + kind: KIND_PROBE, + flags: FLAG_ACK_REQUEST, + seq, + eui64: identity.eui64, + nwk: identity.nwk, + epan: identity.epan, + } + .to_bytes(); + (probe, groups) + }; + for group in groups { + let _ = socket.send_to(&probe, SocketAddr::V6(group)).await; + stats.probes.fetch_add(1, Ordering::Relaxed); + } } - } - targets + }); } fn peer_for(&self, dest: &MacAddress) -> Option { @@ -917,8 +1037,41 @@ impl Backbone { return; }; datagram.extend_from_slice(psdu); - for target in self.broadcast_targets() { + + // One multicast datagram covers every peer it is known to reach; the others + // (no announced address, or probes unanswered) get a unicast copy with the + // same packet number, so a peer that receives both keeps one + let (groups, copies) = { + let shared = self.shared.lock(); + let mut groups: Vec = Vec::new(); + let mut copies: Vec = Vec::new(); + for peer in shared.peers.values() { + match peer.mcast { + Some(group) if peer.multicast_ok() => { + if !groups.contains(&group) { + groups.push(group); + } + } + _ => copies.push(peer.addr), + } + } + // Static peers that have not identified themselves yet + for addr in &self.static_peers { + if !shared.peers.values().any(|p| p.addr == *addr) { + copies.push(*addr); + } + } + (groups, copies) + }; + + self.stats.broadcasts.fetch_add(1, Ordering::Relaxed); + for group in groups { + let _ = self.socket.send_to(&datagram, SocketAddr::V6(group)).await; + self.stats.multicast.fetch_add(1, Ordering::Relaxed); + } + for target in copies { let _ = self.socket.send_to(&datagram, self.target(target)).await; + self.stats.unicast_copies.fetch_add(1, Ordering::Relaxed); } } @@ -1007,7 +1160,10 @@ impl RadioPhy for TrelPhy

{ if let Some(backbone) = &self.backbone { if mac.dest.is_broadcast() { - backbone.send_broadcast(&frame.psdu).await; + // Beacon requests and other MAC commands mean nothing to backbone peers + if mac.frame_type == MAC_FRAME_DATA { + backbone.send_broadcast(&frame.psdu).await; + } } else if let Some(peer) = backbone.peer_for(&mac.dest) { if backbone .send_unicast(peer, &frame.psdu, mac.ack_request) @@ -1096,6 +1252,18 @@ impl RadioPhy for TrelPhy

{ } impl TrelPhy

{ + /// Datagram counters since start, for diagnostics. + pub fn stats(&self) -> Option { + let stats = &self.backbone.as_ref()?.stats; + Some(format!( + "broadcasts={} multicast={} unicast_copies={} probes={}", + stats.broadcasts.load(Ordering::Relaxed), + stats.multicast.load(Ordering::Relaxed), + stats.unicast_copies.load(Ordering::Relaxed), + stats.probes.load(Ordering::Relaxed), + )) + } + /// Peers currently known, for diagnostics. pub fn peers(&self) -> Vec { self.shared @@ -1109,7 +1277,17 @@ impl TrelPhy

{ p.addr, p.source, p.lqi(), - p.mcast.map_or_else(|| "-".to_string(), |mc| mc.to_string()), + p.mcast.map_or_else( + || "-".to_string(), + |mc| format!( + "{mc} {}", + if p.multicast_ok() { + "ok" + } else { + "unconfirmed" + } + ) + ), p.last_rx.map(|t| t.elapsed()) ) }) @@ -1175,6 +1353,7 @@ impl MacAddress { } struct MacHeader { + frame_type: u8, ack_request: bool, dest: MacAddress, source: MacAddress, @@ -1213,6 +1392,7 @@ impl MacHeader { let dest = read_addr(dest_mode, true)?; let source = read_addr(src_mode, !pan_compression)?; Some(Self { + frame_type: (fcf & 0b111) as u8, ack_request, dest, source, diff --git a/crates/ziggurat-phy-trel/tests/loopback.rs b/crates/ziggurat-phy-trel/tests/loopback.rs index 0128f9d..21b3501 100644 --- a/crates/ziggurat-phy-trel/tests/loopback.rs +++ b/crates/ziggurat-phy-trel/tests/loopback.rs @@ -120,6 +120,7 @@ async fn node( interface: None, prefer_ipv4: false, mcast_port: 0, + mcast_block_file: None, deny: DenyList::default(), }, ) diff --git a/crates/ziggurat-server/src/main.rs b/crates/ziggurat-server/src/main.rs index 31d08aa..af8e029 100644 --- a/crates/ziggurat-server/src/main.rs +++ b/crates/ziggurat-server/src/main.rs @@ -1052,6 +1052,11 @@ struct Args { #[arg(long, default_value_t = 19789)] zloe_mcast_port: u16, + /// SPIKE test hook: while this file exists, drop datagrams arriving on the ZLOE + /// multicast group (simulates a network that stopped delivering multicast) + #[arg(long)] + zloe_test_mcast_block_file: Option, + /// SPIKE: disable mDNS discovery of ZLOE peers #[arg(long, alias = "no-mdns")] zloe_no_mdns: bool, @@ -1146,6 +1151,7 @@ fn main() -> Result<(), Box> { interface: args.zloe_interface, prefer_ipv4: args.zloe_ipv4, mcast_port: args.zloe_mcast_port, + mcast_block_file: args.zloe_test_mcast_block_file, deny, }, )); @@ -1159,6 +1165,9 @@ fn main() -> Result<(), Box> { let Some(stack) = server.current_stack() else { continue; }; + if let Some(stats) = stack.radio.stats() { + tracing::info!("[diag] backbone {stats}"); + } for peer in stack.radio.peers() { tracing::info!("[diag] backbone peer {peer}"); } From 033003539e7ba89b72bc2f7a0e9888e08abd9164 Mon Sep 17 00:00:00 2001 From: Igor Yamolov Date: Tue, 10 Nov 2026 16:00:46 +0100 Subject: [PATCH 12/22] SPIKE: IPv4 multicast for ZLOE broadcasts - Join 239.255. on the backbone interface's IPv4 address next to the IPv6 group and announce it as TXT mc4 - Send IPv4 multicast from the unicast socket (IP_MULTICAST_IF) so peers keep learning the unicast port - Probe each group separately; reach each peer through its first confirmed group, IPv6 first or IPv4 first with --zloe-ipv4 - BROADCAST.md: decision to drop the hub and suppression; Wi-Fi peers opt out of multicast with --zloe-mcast-port 0 Co-Authored-By: Claude Opus 5.5 --- crates/ziggurat-phy-trel/BROADCAST.md | 23 +- crates/ziggurat-phy-trel/src/lib.rs | 302 +++++++++++++++++--------- 2 files changed, 223 insertions(+), 102 deletions(-) diff --git a/crates/ziggurat-phy-trel/BROADCAST.md b/crates/ziggurat-phy-trel/BROADCAST.md index 644983c..45972df 100644 --- a/crates/ziggurat-phy-trel/BROADCAST.md +++ b/crates/ziggurat-phy-trel/BROADCAST.md @@ -1,8 +1,10 @@ # ZLOE broadcast delivery: design and gap analysis Status: design for the spike (zigpy/ziggurat#84). Step 1 is built: multicast -announcement, broadcasts by multicast, the probe, and per-peer unicast copies as the -fallback (see "Step 1 as built"). The hub and suppression are not built. +announcement (IPv6 and IPv4), broadcasts by multicast, the probe, and per-peer unicast +copies as the fallback (see "Step 1 as built"). Decision 2026-11-10: no hub and no +suppression. Peers on links where multicast is unreliable (Wi-Fi) run with +`--zloe-mcast-port 0`: they announce no group and get unicast copies. Styleguide: the repository has no written styleguide; its conventions are enforced by `.pre-commit-config.yaml` (`cargo fmt --all`, `cargo clippy --all-targets @@ -71,7 +73,7 @@ limit of the spike; **open** = needs measurement. |---|---|---|---| | M1 | Nodes listen on different UDP ports (two nodes on one host on the bench); multicast needs one shared port. | must | Second socket on the multicast port with address and port reuse (built). Each node announces its group and port over mDNS; a sender uses one datagram per distinct announced address, normally one. Multicast is sent from the unicast socket, so receivers learn the sender's unicast address from it. | | M2 | Link-local multicast is per interface; without `--zloe-interface` the interface is unknown. | must | Multicast only with `--zloe-interface` (built). A node without it announces no multicast address, so peers reach it through the fallback. | -| M3 | IPv4 multicast cannot be joined on the dual-stack socket. | accepted | IPv6 multicast only. IPv4-only links use the fallback. | +| M3 | IPv4 multicast cannot be joined on the dual-stack socket. | resolved | Separate IPv4 socket joins `239.255.h.h` on the backbone interface's address; sending uses the unicast socket with `IP_MULTICAST_IF` set, so peers still see the unicast port. | | M4 | Multicast can silently stop (snooping switch without a querier, Wi-Fi isolation), and can fail in one direction only. | must | Probe: one multicast datagram per 5 s carries the ack-request flag; peers answer by unicast. A peer that misses two probes is unconfirmed, and the sender adds the fallback. Detection ≤ 10 s, well inside the 45 s neighbour ageing. | | M5 | Wi-Fi access points buffer multicast for sleeping clients and send it at the lowest rate. Expected +100–300 ms on broadcasts crossing the backbone. | open | Measure with a Wi-Fi satellite. Unicast is unaffected. | | M6 | Multicast datagrams are unacknowledged. | covered | Each node receives up to N−1 copies of a flood (every relay), and Zigbee re-sends when passive acks are missing. | @@ -123,7 +125,9 @@ multicast-hostile network show the fallback traffic is a problem. - Multicast group: `ff12::5a4c:hhhh:hhhh` (transient, link-local scope), `h` = low 32 bits of the extended PAN ID. - Multicast port: `--zloe-mcast-port`, default 19789, 0 disables multicast. -- mDNS TXT `mc=[group]:port`: where the node listens for multicast. +- mDNS TXT `mc=[group]:port`: where the node listens for IPv6 multicast. +- IPv4 group: `239.255.h.h` (organization-local, RFC 2365), `h` = last two bytes of the + extended PAN ID; TXT `mc4=239.255.h.h:port`. - No new header fields: the original sender is already in the header, a forwarded unicast's destination is in the inner 802.15.4 header. @@ -170,6 +174,17 @@ Bench (coordinator and satellite, `enp4s0f0`, IPv6 link-local): ≤ 10 s M4 aimed at; still well inside the 45 s neighbour ageing. Broadcasts sent in that gap reach the peer only through Zigbee's own retries. +## IPv4 multicast + +- Each node joins both groups when it can: IPv6 needs the dual-stack socket, IPv4 + needs an IPv4 address on `--zloe-interface`. It announces each group it joined. +- Each group is probed separately; a peer is reached through its first confirmed group + in preference order: IPv6 then IPv4, or IPv4 then IPv6 with `--zloe-ipv4`. +- Bench (`--zloe-ipv4` on both nodes): both groups joined and confirmed within 10 s, + broadcasts went to `239.255.124.18:19789`, unicast addresses unaffected. +- Not covered: an IPv4-only host (no dual-stack socket) is untested; the test hook + blocks both groups at once, so per-family failover is untested. + ## Behaviour ``` diff --git a/crates/ziggurat-phy-trel/src/lib.rs b/crates/ziggurat-phy-trel/src/lib.rs index e760aae..85159d9 100644 --- a/crates/ziggurat-phy-trel/src/lib.rs +++ b/crates/ziggurat-phy-trel/src/lib.rs @@ -25,7 +25,7 @@ //! devices at the hardware level; see the spike notes. use std::collections::{HashMap, HashSet, VecDeque}; -use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, SocketAddrV6}; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, SocketAddrV4, SocketAddrV6}; use std::sync::Arc; use std::sync::atomic::{AtomicU32, AtomicU64, Ordering}; use std::time::{Duration, Instant}; @@ -150,9 +150,7 @@ struct Peer { alternates: Vec, /// The multicast group and port the peer listens on (mDNS TXT `mc`), scoped to /// our backbone interface. `None`: the peer cannot be reached by multicast. - mcast: Option, - /// When the peer last acked one of our multicast probes - mcast_acked: Option, + mcast: [Group; 2], /// The confirmation state last logged, to log each change once mcast_logged_ok: bool, } @@ -166,19 +164,19 @@ impl Peer { last_rx: None, health: 1.0, alternates: Vec::new(), - mcast: None, - mcast_acked: None, + mcast: [Group::default(), Group::default()], mcast_logged_ok: false, } } - /// The peer announces a multicast address and has acked a recent probe to it: our - /// multicast reaches it. - fn multicast_ok(&self) -> bool { - self.mcast.is_some() - && self - .mcast_acked - .is_some_and(|t| t.elapsed() < MCAST_CONFIRM_WINDOW) + /// The group our multicast reaches this peer through, by preference: a group the + /// peer announced and has acked a recent probe on. + fn multicast_group(&self, prefer_ipv4: bool) -> Option { + let order = if prefer_ipv4 { [V4, V6] } else { [V6, V4] }; + order + .into_iter() + .find(|&f| self.mcast[f].confirmed()) + .and_then(|f| self.mcast[f].addr) } /// The current address went unanswered: move to the next advertised one. @@ -215,6 +213,28 @@ impl Peer { /// Our identity on the backbone: complete once both the radio configuration (address, /// channel) and the network's extended PAN ID are known. +/// Index into per-family multicast state. +const V6: usize = 0; +const V4: usize = 1; + +/// One multicast group a peer announced, and whether our probes reach it there. +#[derive(Debug, Clone, Copy, Default)] +struct Group { + /// Scoped to our backbone interface (IPv6) or plain (IPv4) + addr: Option, + /// When the peer last acked one of our probes on this group + acked: Option, +} + +impl Group { + fn confirmed(&self) -> bool { + self.addr.is_some() + && self + .acked + .is_some_and(|t| t.elapsed() < MCAST_CONFIRM_WINDOW) + } +} + #[derive(Debug, Clone, Copy)] struct Identity { eui64: Eui64, @@ -243,8 +263,8 @@ struct Shared { resolved: HashMap, netinfo_provider: Option, netinfo_waiter: Option>>, - /// Packet numbers of our latest multicast probes - probes: VecDeque, + /// Packet numbers of our latest multicast probes, with their family + probes: VecDeque<(u32, usize)>, } /// Datagram counters, for the bench. @@ -277,6 +297,7 @@ struct Backbone { mcast_port: u16, /// The task reading our multicast socket; replaced when the network changes mcast_reader: Mutex>>, + mcast4_reader: Mutex>>, } pub struct TrelPhy { @@ -361,6 +382,7 @@ impl TrelPhy

{ dual_stack, mcast_port: config.mcast_port, mcast_reader: Mutex::new(None), + mcast4_reader: Mutex::new(None), }; backbone.spawn_udp_reader(Arc::clone(&backbone.socket), None); backbone.spawn_mdns_browser(); @@ -473,7 +495,8 @@ struct Resolved { nwk: Option, /// Usable addresses, best first, with their rank candidates: Vec<((u8, u8), SocketAddr)>, - mcast: Option, + /// Announced groups: [IPv6 (`mc`), IPv4 (`mc4`)] + mcast: [Option; 2], } /// Create or update the peer for a resolved node on our network. @@ -505,10 +528,17 @@ fn apply_resolved(shared: &mut Shared, eui64: Eui64) { .find(|(_, a)| *a == peer.addr) .map(|(k, _)| *k); peer.alternates = resolved.candidates.iter().map(|(_, a)| *a).collect(); - if peer.mcast != resolved.mcast { - tracing::info!("ZLOE: peer {eui64:?} multicast: {:?}", resolved.mcast); - peer.mcast = resolved.mcast; - peer.mcast_acked = None; + for family in [V6, V4] { + if peer.mcast[family].addr != resolved.mcast[family] { + tracing::info!( + "ZLOE: peer {eui64:?} multicast: {:?}", + resolved.mcast[family] + ); + peer.mcast[family] = Group { + addr: resolved.mcast[family], + acked: None, + }; + } } if peer.last_rx.is_none() && current_key.is_none_or(|k| best_key < k) { tracing::info!("ZLOE: mDNS peer {eui64:?} moved to {addr}"); @@ -615,10 +645,15 @@ impl Backbone { } else { // Not a unicast we are waiting on: an answer to a probe? let mut shared = shared.lock(); - if shared.probes.contains(&header.seq) + let family = shared + .probes + .iter() + .find(|(seq, _)| *seq == header.seq) + .map(|(_, f)| *f); + if let Some(family) = family && let Some(peer) = shared.peers.get_mut(&header.eui64) { - peer.mcast_acked = Some(Instant::now()); + peer.mcast[family].acked = Some(Instant::now()); } } continue; @@ -758,15 +793,24 @@ impl Backbone { .get_property_val_str("nwk") .and_then(|s| Nwk::try_from_hex(s).ok()); // Usable only on a known interface: the group is link-local - let mcast = info + // Usable only on a known interface: groups are link-local + // (IPv6) or bound to the interface's subnet (IPv4) + let mc6 = info .get_property_val_str("mc") .and_then(|s| s.parse::().ok()) .filter(|mc| mc.ip().is_multicast()) .zip(local_net) .map(|(mut mc, net)| { mc.set_scope_id(net.index); - mc + SocketAddr::V6(mc) }); + let mc4 = info + .get_property_val_str("mc4") + .and_then(|s| s.parse::().ok()) + .filter(|mc| mc.ip().is_multicast()) + .filter(|_| local_net.is_some_and(|n| n.v4.is_some())) + .map(SocketAddr::V4); + let mcast = [mc6, mc4]; // mDNS lists every address of every interface (Docker bridges, // loopback, ...): pick deterministically let candidates = candidate_addresses( @@ -814,7 +858,7 @@ impl Backbone { if let Some(peer) = shared.peers.get_mut(&eui64) { // A peer kept alive by its own traffic loses what only // mDNS told us - peer.mcast = None; + peer.mcast = [Group::default(), Group::default()]; } if shared .peers @@ -831,47 +875,91 @@ impl Backbone { }); } - /// Join the network's multicast group and read from it. `None` when multicast is - /// off, the backbone interface is unknown, or the host has no IPv6. - fn listen_multicast(&self, epan: Eui64) -> Option { - use socket2::{Domain, Protocol, Socket, Type}; + /// Join the network's multicast groups and read from them: `[IPv6, IPv4]`, each + /// `None` when multicast is off, the backbone interface (or its address of that + /// family) is unknown, or joining fails. + fn listen_multicast(&self, epan: Eui64) -> [Option; 2] { + use socket2::{Domain, Protocol, SockRef, Socket, Type}; - if let Some(old) = self.mcast_reader.lock().take() { - old.abort(); // closes the socket, leaving the previous group + for reader in [&self.mcast_reader, &self.mcast4_reader] { + if let Some(old) = reader.lock().take() { + old.abort(); // closes the socket, leaving the previous group + } } - if self.mcast_port == 0 || !self.dual_stack { - return None; + if self.mcast_port == 0 { + return [None, None]; } let Some(net) = self.local_net else { tracing::info!("ZLOE: no multicast without --zloe-interface"); - return None; + return [None, None]; }; - let group = multicast_group(epan); - let open = || -> std::io::Result { - let socket = Socket::new(Domain::IPV6, Type::DGRAM, Some(Protocol::UDP))?; - socket.set_only_v6(true)?; - // Several nodes on one host (the bench) share the port + // Several nodes on one host (the bench) share the port + let open = |domain: Domain, bind: SocketAddr| -> std::io::Result { + let socket = Socket::new(domain, Type::DGRAM, Some(Protocol::UDP))?; + if domain == Domain::IPV6 { + socket.set_only_v6(true)?; + } socket.set_reuse_address(true)?; socket.set_reuse_port(true)?; - socket.bind(&SocketAddr::from((Ipv6Addr::UNSPECIFIED, self.mcast_port)).into())?; - socket.join_multicast_v6(&group, net.index)?; + socket.bind(&bind.into())?; + Ok(socket) + }; + let start = |socket: Socket| -> std::io::Result> { socket.set_nonblocking(true)?; - UdpSocket::from_std(socket.into()) + Ok(Arc::new(UdpSocket::from_std(socket.into())?)) }; - match open() { - Ok(socket) => { - let mc = SocketAddrV6::new(group, self.mcast_port, 0, net.index); - tracing::info!("ZLOE: listening on multicast group {mc}"); - *self.mcast_reader.lock() = - Some(self.spawn_udp_reader(Arc::new(socket), self.mcast_block_file.clone())); - Some(mc) + + let mc6 = self.dual_stack.then(|| { + let group = multicast_group(epan); + let joined = open( + Domain::IPV6, + (Ipv6Addr::UNSPECIFIED, self.mcast_port).into(), + ) + .and_then(|s| s.join_multicast_v6(&group, net.index).map(|()| s)) + .and_then(start); + match joined { + Ok(socket) => { + let mc = SocketAddrV6::new(group, self.mcast_port, 0, net.index); + tracing::info!("ZLOE: listening on multicast group {mc}"); + *self.mcast_reader.lock() = + Some(self.spawn_udp_reader(socket, self.mcast_block_file.clone())); + Some(SocketAddr::V6(mc)) + } + Err(e) => { + tracing::warn!("ZLOE: cannot join multicast group {group}: {e}"); + None + } } - Err(e) => { - tracing::warn!("ZLOE: cannot join multicast group {group}: {e}"); - None + }); + + let mc4 = net.v4.map(|(iface_ip, _)| { + let group = multicast_group_v4(epan); + let joined = open( + Domain::IPV4, + (Ipv4Addr::UNSPECIFIED, self.mcast_port).into(), + ) + .and_then(|s| s.join_multicast_v4(&group, &iface_ip).map(|()| s)) + .and_then(start); + // Our IPv4 multicast leaves through the unicast socket (so peers see our + // unicast port), out of the backbone interface + let sending = SockRef::from(&*self.socket).set_multicast_if_v4(&iface_ip); + match (joined, sending) { + (Ok(socket), Ok(())) => { + let mc = SocketAddrV4::new(group, self.mcast_port); + tracing::info!("ZLOE: listening on multicast group {mc}"); + *self.mcast4_reader.lock() = + Some(self.spawn_udp_reader(socket, self.mcast_block_file.clone())); + Some(SocketAddr::V4(mc)) + } + (Err(e), _) | (_, Err(e)) => { + tracing::warn!("ZLOE: cannot use IPv4 multicast group {group}: {e}"); + None + } } - } + }); + + [mc6.flatten(), mc4.flatten()] } fn advertise(&self, identity: Identity) { @@ -903,9 +991,12 @@ impl Backbone { // `mc`: where we listen for multicast. Absent when we cannot, so peers know // to reach us by unicast only - if let Some(mc) = mcast { + if let [Some(SocketAddr::V6(mc)), _] = mcast { properties.insert("mc".to_string(), format!("[{}]:{}", mc.ip(), mc.port())); } + if let [_, Some(mc4)] = mcast { + properties.insert("mc4".to_string(), mc4.to_string()); + } // Addresses (IPv6 link-local and IPv4) come from the interfaces mDNS runs on, // which --zloe-interface already narrows to one @@ -933,33 +1024,39 @@ impl Backbone { let shared = Arc::clone(&self.shared); let seq = Arc::clone(&self.seq); let stats = Arc::clone(&self.stats); + let prefer_ipv4 = self.prefer_ipv4; + let dual_stack = self.dual_stack; tokio::spawn(async move { let mut tick = tokio::time::interval(PROBE_INTERVAL); loop { tick.tick().await; - let (probe, groups) = { + let probes = { let mut shared = shared.lock(); let Some(identity) = shared.identity else { continue; }; - let mut groups: Vec = Vec::new(); + let mut groups: Vec<(usize, SocketAddr)> = Vec::new(); for (eui64, peer) in &mut shared.peers { - let Some(group) = peer.mcast else { continue }; - if !groups.contains(&group) { - groups.push(group); + for family in [V6, V4] { + if let Some(group) = peer.mcast[family].addr + && !groups.contains(&(family, group)) + { + groups.push((family, group)); + } } - let ok = peer.multicast_ok(); - if ok != peer.mcast_logged_ok { - peer.mcast_logged_ok = ok; - if ok { - tracing::info!("ZLOE: multicast reaches {eui64:?}"); - } else { - tracing::warn!( + let ok = peer.multicast_group(prefer_ipv4); + if ok.is_some() != peer.mcast_logged_ok { + peer.mcast_logged_ok = ok.is_some(); + match ok { + Some(group) => { + tracing::info!("ZLOE: multicast reaches {eui64:?} via {group}"); + } + None => tracing::warn!( "ZLOE: multicast does not reach {eui64:?}, \ sending it unicast copies" - ); + ), } } } @@ -967,24 +1064,31 @@ impl Backbone { continue; } - let seq = seq.fetch_add(1, Ordering::Relaxed); - shared.probes.push_back(seq); - if shared.probes.len() > 4 { + // One probe per group, so an ack tells which family works + let probes: Vec<(SocketAddr, Vec)> = groups + .into_iter() + .map(|(family, group)| { + let seq = seq.fetch_add(1, Ordering::Relaxed); + shared.probes.push_back((seq, family)); + let probe = WireHeader { + kind: KIND_PROBE, + flags: FLAG_ACK_REQUEST, + seq, + eui64: identity.eui64, + nwk: identity.nwk, + epan: identity.epan, + } + .to_bytes(); + (group, probe) + }) + .collect(); + while shared.probes.len() > 8 { shared.probes.pop_front(); } - let probe = WireHeader { - kind: KIND_PROBE, - flags: FLAG_ACK_REQUEST, - seq, - eui64: identity.eui64, - nwk: identity.nwk, - epan: identity.epan, - } - .to_bytes(); - (probe, groups) + probes }; - for group in groups { - let _ = socket.send_to(&probe, SocketAddr::V6(group)).await; + for (group, probe) in probes { + let _ = socket.send_to(&probe, to_family(group, dual_stack)).await; stats.probes.fetch_add(1, Ordering::Relaxed); } } @@ -1043,16 +1147,16 @@ impl Backbone { // same packet number, so a peer that receives both keeps one let (groups, copies) = { let shared = self.shared.lock(); - let mut groups: Vec = Vec::new(); + let mut groups: Vec = Vec::new(); let mut copies: Vec = Vec::new(); for peer in shared.peers.values() { - match peer.mcast { - Some(group) if peer.multicast_ok() => { + match peer.multicast_group(self.prefer_ipv4) { + Some(group) => { if !groups.contains(&group) { groups.push(group); } } - _ => copies.push(peer.addr), + None => copies.push(peer.addr), } } // Static peers that have not identified themselves yet @@ -1066,7 +1170,7 @@ impl Backbone { self.stats.broadcasts.fetch_add(1, Ordering::Relaxed); for group in groups { - let _ = self.socket.send_to(&datagram, SocketAddr::V6(group)).await; + let _ = self.socket.send_to(&datagram, self.target(group)).await; self.stats.multicast.fetch_add(1, Ordering::Relaxed); } for target in copies { @@ -1277,17 +1381,12 @@ impl TrelPhy

{ p.addr, p.source, p.lqi(), - p.mcast.map_or_else( - || "-".to_string(), - |mc| format!( - "{mc} {}", - if p.multicast_ok() { - "ok" - } else { - "unconfirmed" - } - ) - ), + [V6, V4] + .into_iter() + .filter_map(|f| p.mcast[f].addr.map(|a| (a, p.mcast[f].confirmed()))) + .map(|(a, ok)| format!("{a} {}", if ok { "ok" } else { "unconfirmed" })) + .collect::>() + .join(", "), p.last_rx.map(|t| t.elapsed()) ) }) @@ -1476,6 +1575,13 @@ fn multicast_group(epan: Eui64) -> Ipv6Addr { ) } +/// The network's IPv4 multicast group, organization-local scope (RFC 2365), with the +/// last two bytes of the extended PAN ID: `239.255.h.h`. +fn multicast_group_v4(epan: Eui64) -> Ipv4Addr { + let e = epan.to_be_bytes(); + Ipv4Addr::new(239, 255, e[6], e[7]) +} + /// IPv4 targets on a dual-stack IPv6 socket must be v4-mapped. fn to_family(addr: SocketAddr, dual_stack: bool) -> SocketAddr { match addr { From 169049211933f172fdbfe56c63e994730e6036c3 Mon Sep 17 00:00:00 2001 From: Igor Yamolov Date: Tue, 10 Nov 2026 16:10:48 +0100 Subject: [PATCH 13/22] SPIKE: test hooks for ZLOE multicast families and IPv4-only hosts - The multicast block file blocks one group when it contains "v4" or "v6", both when empty - --zloe-test-ipv4-only behaves like a host without IPv6; such a host ignores IPv6 peer addresses and groups - Bench: per-family failover kept the peer on multicast through the other group; an IPv4-only satellite bootstraps and is reached through the IPv4 group Co-Authored-By: Claude Opus 5.5 --- crates/ziggurat-phy-trel/BROADCAST.md | 11 +++++- crates/ziggurat-phy-trel/src/lib.rs | 43 +++++++++++++++------- crates/ziggurat-phy-trel/tests/loopback.rs | 1 + crates/ziggurat-server/src/main.rs | 8 +++- 4 files changed, 47 insertions(+), 16 deletions(-) diff --git a/crates/ziggurat-phy-trel/BROADCAST.md b/crates/ziggurat-phy-trel/BROADCAST.md index 45972df..88f2a63 100644 --- a/crates/ziggurat-phy-trel/BROADCAST.md +++ b/crates/ziggurat-phy-trel/BROADCAST.md @@ -182,8 +182,15 @@ Bench (coordinator and satellite, `enp4s0f0`, IPv6 link-local): in preference order: IPv6 then IPv4, or IPv4 then IPv6 with `--zloe-ipv4`. - Bench (`--zloe-ipv4` on both nodes): both groups joined and confirmed within 10 s, broadcasts went to `239.255.124.18:19789`, unicast addresses unaffected. -- Not covered: an IPv4-only host (no dual-stack socket) is untested; the test hook - blocks both groups at once, so per-family failover is untested. +- Test hooks: `--zloe-test-mcast-block-file` blocks both groups while the file is + empty, or one group when it contains `v4` or `v6`; `--zloe-test-ipv4-only` behaves + like a host without IPv6. +- Per-family failover (both nodes `--zloe-ipv4`, IPv4 blocked on the satellite): the + satellite's IPv4 group went unconfirmed, the coordinator kept reaching it through + the IPv6 group (no unicast copies), IPv4 came back after unblocking. +- IPv4-only satellite: bootstrap over IPv4, announces only `mc4`, the dual-stack + coordinator reaches it through the IPv4 group with no unicast copies, neighbour + cost 1/1. IPv6 groups are ignored without IPv6. ## Behaviour diff --git a/crates/ziggurat-phy-trel/src/lib.rs b/crates/ziggurat-phy-trel/src/lib.rs index 85159d9..5e00f2d 100644 --- a/crates/ziggurat-phy-trel/src/lib.rs +++ b/crates/ziggurat-phy-trel/src/lib.rs @@ -104,6 +104,8 @@ pub struct TrelConfig { /// Test hook: while this file exists, datagrams arriving on the multicast group /// are dropped, as if the network had stopped delivering multicast to us. pub mcast_block_file: Option, + /// Test hook: behave like a host without IPv6 (IPv4 socket, no IPv6 peers or group) + pub ipv4_only: bool, pub deny: DenyList, } @@ -325,7 +327,7 @@ impl TrelPhy

{ None } else { let local_net = config.interface.as_deref().map(local_net).transpose()?; - let (socket, dual_stack) = bind_socket(config.port)?; + let (socket, dual_stack) = bind_socket(config.port, config.ipv4_only)?; let socket = Arc::new(socket); tracing::info!( "ZLOE backbone listening on UDP {} ({}, {}, prefer {})", @@ -575,13 +577,17 @@ fn update_identity(shared: &mut Shared) -> Option { /// One dual-stack IPv6 socket (IPv4 arrives v4-mapped); IPv4-only when the host has no /// IPv6. -fn bind_socket(port: u16) -> std::io::Result<(UdpSocket, bool)> { +fn bind_socket(port: u16, ipv4_only: bool) -> std::io::Result<(UdpSocket, bool)> { use socket2::{Domain, Protocol, Socket, Type}; - let v6 = Socket::new(Domain::IPV6, Type::DGRAM, Some(Protocol::UDP)).and_then(|s| { - s.set_only_v6(false)?; - s.bind(&SocketAddr::from((Ipv6Addr::UNSPECIFIED, port)).into())?; - Ok(s) - }); + let v6 = if ipv4_only { + Err(std::io::Error::other("IPv4 only requested")) + } else { + Socket::new(Domain::IPV6, Type::DGRAM, Some(Protocol::UDP)).and_then(|s| { + s.set_only_v6(false)?; + s.bind(&SocketAddr::from((Ipv6Addr::UNSPECIFIED, port)).into())?; + Ok(s) + }) + }; let (socket, dual_stack) = match v6 { Ok(s) => (s, true), Err(e) => { @@ -612,7 +618,7 @@ impl Backbone { fn spawn_udp_reader( &self, rx: Arc, - block_file: Option, + block: Option<(std::path::PathBuf, &'static str)>, ) -> tokio::task::JoinHandle<()> { let socket = Arc::clone(&self.socket); let shared = Arc::clone(&self.shared); @@ -630,7 +636,11 @@ impl Backbone { continue; } }; - if block_file.as_deref().is_some_and(std::path::Path::exists) { + // Test hook: an empty file blocks every group, "v4"/"v6" just one + if let Some((file, family)) = &block + && let Ok(content) = std::fs::read_to_string(file) + && (content.trim().is_empty() || content.contains(family)) + { continue; } let Some(header) = WireHeader::parse(&buf[..len]) else { @@ -774,6 +784,7 @@ impl Backbone { }; let shared = Arc::clone(&self.shared); let local_net = self.local_net; + let dual_stack = self.dual_stack; let prefer_ipv4 = self.prefer_ipv4; tokio::spawn(async move { @@ -798,7 +809,7 @@ impl Backbone { let mc6 = info .get_property_val_str("mc") .and_then(|s| s.parse::().ok()) - .filter(|mc| mc.ip().is_multicast()) + .filter(|mc| mc.ip().is_multicast() && dual_stack) .zip(local_net) .map(|(mut mc, net)| { mc.set_scope_id(net.index); @@ -818,6 +829,7 @@ impl Backbone { info.get_port(), local_net, prefer_ipv4, + dual_stack, ); let Some(&(_, addr)) = candidates.first() else { continue; @@ -875,6 +887,10 @@ impl Backbone { }); } + fn block_hook(&self, family: &'static str) -> Option<(std::path::PathBuf, &'static str)> { + self.mcast_block_file.clone().map(|f| (f, family)) + } + /// Join the network's multicast groups and read from them: `[IPv6, IPv4]`, each /// `None` when multicast is off, the backbone interface (or its address of that /// family) is unknown, or joining fails. @@ -923,7 +939,7 @@ impl Backbone { let mc = SocketAddrV6::new(group, self.mcast_port, 0, net.index); tracing::info!("ZLOE: listening on multicast group {mc}"); *self.mcast_reader.lock() = - Some(self.spawn_udp_reader(socket, self.mcast_block_file.clone())); + Some(self.spawn_udp_reader(socket, self.block_hook("v6"))); Some(SocketAddr::V6(mc)) } Err(e) => { @@ -949,7 +965,7 @@ impl Backbone { let mc = SocketAddrV4::new(group, self.mcast_port); tracing::info!("ZLOE: listening on multicast group {mc}"); *self.mcast4_reader.lock() = - Some(self.spawn_udp_reader(socket, self.mcast_block_file.clone())); + Some(self.spawn_udp_reader(socket, self.block_hook("v4"))); Some(SocketAddr::V4(mc)) } (Err(e), _) | (_, Err(e)) => { @@ -1511,6 +1527,7 @@ fn candidate_addresses( port: u16, local_net: Option, prefer_ipv4: bool, + dual_stack: bool, ) -> Vec<((u8, u8), SocketAddr)> { let mut ranked: Vec<((u8, u8), SocketAddr)> = addrs .iter() @@ -1518,7 +1535,7 @@ fn candidate_addresses( mdns_sd::ScopedIp::V6(v6) => { let ip = *v6.addr(); let index = v6.scope_id().index; - if ip.is_loopback() { + if ip.is_loopback() || !dual_stack { return None; } let link_local = (ip.segments()[0] & 0xffc0) == 0xfe80; diff --git a/crates/ziggurat-phy-trel/tests/loopback.rs b/crates/ziggurat-phy-trel/tests/loopback.rs index 21b3501..3761417 100644 --- a/crates/ziggurat-phy-trel/tests/loopback.rs +++ b/crates/ziggurat-phy-trel/tests/loopback.rs @@ -121,6 +121,7 @@ async fn node( prefer_ipv4: false, mcast_port: 0, mcast_block_file: None, + ipv4_only: false, deny: DenyList::default(), }, ) diff --git a/crates/ziggurat-server/src/main.rs b/crates/ziggurat-server/src/main.rs index af8e029..006b159 100644 --- a/crates/ziggurat-server/src/main.rs +++ b/crates/ziggurat-server/src/main.rs @@ -1053,10 +1053,15 @@ struct Args { zloe_mcast_port: u16, /// SPIKE test hook: while this file exists, drop datagrams arriving on the ZLOE - /// multicast group (simulates a network that stopped delivering multicast) + /// multicast groups (simulates a network that stopped delivering multicast). An + /// empty file blocks both groups; containing "v4" or "v6" blocks only that one #[arg(long)] zloe_test_mcast_block_file: Option, + /// SPIKE test hook: behave like a host without IPv6 + #[arg(long)] + zloe_test_ipv4_only: bool, + /// SPIKE: disable mDNS discovery of ZLOE peers #[arg(long, alias = "no-mdns")] zloe_no_mdns: bool, @@ -1152,6 +1157,7 @@ fn main() -> Result<(), Box> { prefer_ipv4: args.zloe_ipv4, mcast_port: args.zloe_mcast_port, mcast_block_file: args.zloe_test_mcast_block_file, + ipv4_only: args.zloe_test_ipv4_only, deny, }, )); From 06c967804ec3301c273fd6c1b3687226177ec145 Mon Sep 17 00:00:00 2001 From: Igor Yamolov Date: Tue, 10 Nov 2026 16:36:08 +0100 Subject: [PATCH 14/22] SPIKE: satellites broadcast Device_annce on start A board that rejoins as a satellite under a new NWK address kept its old address in the other nodes' neighbor tables (keyed by EUI64), so its Link Status never mapped and its links stayed at cost 0. Announce like any router after joining. Co-Authored-By: Claude Opus 5.5 --- crates/ziggurat-server/src/satellite.rs | 28 +++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/crates/ziggurat-server/src/satellite.rs b/crates/ziggurat-server/src/satellite.rs index df75ead..0c9d619 100644 --- a/crates/ziggurat-server/src/satellite.rs +++ b/crates/ziggurat-server/src/satellite.rs @@ -78,6 +78,7 @@ async fn start_router( .await .map_err(|_| "start_network failed")?; + announce(server, nwk, ieee); tracing::info!("Satellite: router is up"); Ok(()) } @@ -278,3 +279,30 @@ fn spawn_zdp_responder(server: &std::sync::Arc) { } }); } + +/// ZDP Device_annce (spec 2.4.3.1.11), as any router does after (re)joining: maps our +/// EUI64 to our (possibly new) NWK address in every node's address and neighbor +/// tables, and lets the coordinator's client (ZHA) learn about the satellite. +fn announce(server: &std::sync::Arc, nwk: Nwk, ieee: Eui64) { + use ziggurat_driver::zigbee_stack::TxPriority; + + let Some(stack) = server.current_stack() else { + return; + }; + // Capability: router, mains powered, receiver on when idle, allocate address + let mut payload = vec![0x00]; + payload.extend_from_slice(&nwk.0.to_le_bytes()); + payload.extend_from_slice(&ieee.0); + payload.push(0x8E); + let result = stack.send_aps_broadcast( + Nwk(0xFFFD), + 0x0000, + 0x0013, + 0, + 0, + 30, + payload, + TxPriority::StackCritical, + ); + tracing::info!("Satellite: Device_annce for {nwk:?}: {:?}", result.map(|_| ())); +} From 902cd553731bc570ed5d876921d057e51aaa8bf7 Mon Sep 17 00:00:00 2001 From: Igor Yamolov Date: Tue, 10 Nov 2026 16:48:16 +0100 Subject: [PATCH 15/22] SPIKE: probe only the preferred ZLOE multicast group Probe each peer's preferred group (IPv6, or IPv4 with --zloe-ipv4) and the other group only while the preferred one is absent or unconfirmed. Halves probe traffic (144 -> 72 per 120 s on the 3-node bench); failover to the other family takes one more probe interval. Co-Authored-By: Claude Opus 5.5 --- crates/ziggurat-phy-trel/BROADCAST.md | 13 +++++++++++++ crates/ziggurat-phy-trel/src/lib.rs | 16 ++++++++++++---- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/crates/ziggurat-phy-trel/BROADCAST.md b/crates/ziggurat-phy-trel/BROADCAST.md index 88f2a63..c11f4ff 100644 --- a/crates/ziggurat-phy-trel/BROADCAST.md +++ b/crates/ziggurat-phy-trel/BROADCAST.md @@ -192,6 +192,19 @@ Bench (coordinator and satellite, `enp4s0f0`, IPv6 link-local): coordinator reaches it through the IPv4 group with no unicast copies, neighbour cost 1/1. IPv6 groups are ignored without IPv6. +## Probe overhead + +- Every node used to probe every announced group (IPv6 and IPv4) every 5 s. With 3 + backbone nodes that was 144 probes (plus ~2 acks each) per 120 s against 38 data + datagrams saved. +- Now a node probes each peer's preferred group only, and the other group only while + the preferred one is absent or unconfirmed: 72 probes per 120 s on the same bench, + still 1.00 data datagram per broadcast. +- Failover to the other group now takes one more probe interval: IPv6 blocked, the + peer got unicast copies after 13 s and was back on multicast via IPv4 5 s later. +- At this traffic level (~0.1 broadcasts/s per node) probes plus acks still outweigh + the saving; piggybacking the ack request on real broadcasts would remove most of it. + ## Behaviour ``` diff --git a/crates/ziggurat-phy-trel/src/lib.rs b/crates/ziggurat-phy-trel/src/lib.rs index 5e00f2d..762b8e7 100644 --- a/crates/ziggurat-phy-trel/src/lib.rs +++ b/crates/ziggurat-phy-trel/src/lib.rs @@ -1055,12 +1055,20 @@ impl Backbone { let mut groups: Vec<(usize, SocketAddr)> = Vec::new(); for (eui64, peer) in &mut shared.peers { - for family in [V6, V4] { - if let Some(group) = peer.mcast[family].addr - && !groups.contains(&(family, group)) - { + // Probe the preferred group only; the other one only while + // the preferred one is absent or unconfirmed, so a peer can + // fail over without doubling the probe traffic + let order = if prefer_ipv4 { [V4, V6] } else { [V6, V4] }; + for family in order { + let Some(group) = peer.mcast[family].addr else { + continue; + }; + if !groups.contains(&(family, group)) { groups.push((family, group)); } + if peer.mcast[family].confirmed() { + break; + } } let ok = peer.multicast_group(prefer_ipv4); if ok.is_some() != peer.mcast_logged_ok { From 430634dfb70b779e58be2757e3cd2c3df5ea1822 Mon Sep 17 00:00:00 2001 From: Igor Yamolov Date: Tue, 10 Nov 2026 17:08:23 +0100 Subject: [PATCH 16/22] SPIKE: satellites answer the ZDP interview themselves zigpy interviews a satellite after its Device_annce, but a satellite has no client to answer ZDP, so the interview never finished and ZHA did not list it. Answer Node_Desc_req (router descriptor) and Active_EP_req (no endpoints) for the node's own address; the test-network coordinator uses the same responder. Co-Authored-By: Claude Opus 5.5 --- crates/ziggurat-server/src/satellite.rs | 72 ++++++++++++++++++------- 1 file changed, 52 insertions(+), 20 deletions(-) diff --git a/crates/ziggurat-server/src/satellite.rs b/crates/ziggurat-server/src/satellite.rs index 0c9d619..fa5d534 100644 --- a/crates/ziggurat-server/src/satellite.rs +++ b/crates/ziggurat-server/src/satellite.rs @@ -78,6 +78,7 @@ async fn start_router( .await .map_err(|_| "start_network failed")?; + spawn_zdp_responder(server, nwk); announce(server, nwk, ieee); tracing::info!("Satellite: router is up"); Ok(()) @@ -207,16 +208,20 @@ pub async fn test_coordinator( .await .map_err(|_| "start_network failed")?; tracing::info!("Test network: coordinator is up"); - spawn_zdp_responder(server); + spawn_zdp_responder(server, Nwk(0x0000)); Ok(()) } -/// SPIKE: zigpy normally answers ZDP requests addressed to the coordinator. Without a -/// client, joiners (ZBOSS) give up when Node_Desc_req for 0x0000 goes unanswered (they -/// need the TC's stack revision), so answer that one ourselves. -fn spawn_zdp_responder(server: &std::sync::Arc) { +/// SPIKE: answer the ZDP requests of a device interview ourselves, for nodes without +/// a client to do it: the test-network coordinator (zigpy's job normally; ZBOSS joiners +/// leave when the coordinator's Node_Desc_req goes unanswered) and satellites (zigpy +/// interviews them after their Device_annce and gives up otherwise). +/// +/// Answers Node_Desc_req and Active_EP_req (no endpoints) for our own address. +fn spawn_zdp_responder(server: &std::sync::Arc, own: Nwk) { use ziggurat_driver::zigbee_stack::{ApsAck, RouteDirective, TxPriority, ZigbeeNotification}; + let coordinator = own == Nwk(0x0000); let mut notifications = server.notification_tx.subscribe(); let server = server.clone(); tokio::spawn(async move { @@ -229,7 +234,7 @@ fn spawn_zdp_responder(server: &std::sync::Arc) { let ZigbeeNotification::ReceivedApsCommand { source, profile_id: 0x0000, - cluster_id: 0x0002, + cluster_id, dst_ep: 0, data, .. @@ -240,32 +245,56 @@ fn spawn_zdp_responder(server: &std::sync::Arc) { let [seq, lo, hi, ..] = data[..] else { continue; }; - if u16::from_le_bytes([lo, hi]) != 0x0000 { + if Nwk(u16::from_le_bytes([lo, hi])) != own { continue; } let Some(stack) = server.current_stack() else { continue; }; - // Node descriptor: coordinator, 2.4 GHz, MAC caps 0x8F, manufacturer 0x0000, - // buffer 82, transfer sizes 128, server mask = primary TC + network manager, - // stack compliance revision 22 - let server_mask: u16 = 0x0001 | 0x0040 | (22 << 9); - let mut rsp = vec![seq, 0x00, 0x00, 0x00, 0x00, 0x40, 0x8F, 0x00, 0x00, 0x52]; - rsp.extend_from_slice(&128u16.to_le_bytes()); - rsp.extend_from_slice(&server_mask.to_le_bytes()); - rsp.extend_from_slice(&128u16.to_le_bytes()); - rsp.push(0x00); + let (rsp_cluster, rsp) = match cluster_id { + // Node descriptor: 2.4 GHz, buffer 82, transfer sizes 128, stack + // compliance revision 22. Coordinator: logical type 0, MAC caps 0x8F, + // server mask primary TC + network manager. Router: type 1, caps 0x8E + 0x0002 => { + let (logical_type, caps, mask) = if coordinator { + (0x00, 0x8F, 0x0001 | 0x0040) + } else { + (0x01, 0x8E, 0x0000) + }; + let server_mask: u16 = mask | (22 << 9); + let mut rsp = vec![ + seq, + 0x00, + lo, + hi, + logical_type, + 0x40, + caps, + 0x00, + 0x00, + 0x52, + ]; + rsp.extend_from_slice(&128u16.to_le_bytes()); + rsp.extend_from_slice(&server_mask.to_le_bytes()); + rsp.extend_from_slice(&128u16.to_le_bytes()); + rsp.push(0x00); + (0x8002, rsp) + } + // Active endpoints: none + 0x0005 => (0x8005, vec![seq, 0x00, lo, hi, 0x00]), + _ => continue, + }; let result = stack.send_aps_unicast( source, 0x0000, - 0x8002, + rsp_cluster, 0, 0, ApsAck::None, None, - 0, + 30, rsp, None, false, @@ -273,7 +302,7 @@ fn spawn_zdp_responder(server: &std::sync::Arc) { RouteDirective::StackDecides, ); tracing::info!( - "Test network: answered Node_Desc_req from {source:?}: {:?}", + "ZDP responder: answered {cluster_id:#06x} from {source:?}: {:?}", result.map(|_| ()) ); } @@ -304,5 +333,8 @@ fn announce(server: &std::sync::Arc, nwk: Nwk, ieee: Eui64) { payload, TxPriority::StackCritical, ); - tracing::info!("Satellite: Device_annce for {nwk:?}: {:?}", result.map(|_| ())); + tracing::info!( + "Satellite: Device_annce for {nwk:?}: {:?}", + result.map(|_| ()) + ); } From e30941e64cb2d9aa5b093f54209394d4971b93fb Mon Sep 17 00:00:00 2001 From: Igor Yamolov Date: Tue, 10 Nov 2026 17:33:03 +0100 Subject: [PATCH 17/22] SPIKE: ZHA topology scan of the coordinator; QUIC security design - Deliver frames addressed to our own NWK address locally: zigpy scans the coordinator's neighbor and routing tables with ZDP to 0x0000, which Ziggurat used to route-discover and time out - Mgmt_Lqi_rsp reports 0x0000 as the coordinator at depth 0 (entries learned from Link Status defaulted to Router, depth 1) - BROADCAST.md: secure ZLOE over QUIC, keys derived from the network key, enrollment codes, sealed multicast; gap analysis and behaviour Co-Authored-By: Claude Opus 5.5 --- .../ziggurat-driver/src/zigbee_stack/nwk.rs | 42 +++++ .../ziggurat-driver/src/zigbee_stack/zdp.rs | 10 +- crates/ziggurat-phy-trel/BROADCAST.md | 171 ++++++++++++++++++ crates/ziggurat-zigbee/src/sends/mod.rs | 6 + 4 files changed, 226 insertions(+), 3 deletions(-) diff --git a/crates/ziggurat-driver/src/zigbee_stack/nwk.rs b/crates/ziggurat-driver/src/zigbee_stack/nwk.rs index 5bea6ad..d8131ff 100644 --- a/crates/ziggurat-driver/src/zigbee_stack/nwk.rs +++ b/crates/ziggurat-driver/src/zigbee_stack/nwk.rs @@ -336,6 +336,41 @@ impl ZigbeeStack { }) } + /// Deliver a frame we addressed to ourselves as if it had been received: the stack + /// handles the ZDP requests it owns (its answers loop back the same way), and the + /// client gets the frame. The send succeeds at once. + fn deliver_locally(&self, nwk_frame: &NwkFrame) -> SendHandle { + use ziggurat_zigbee::aps::frame::{ApsFrame, parse_aps_frame}; + + let slot = Arc::new(super::SendSlot::default()); + let id = self.core().sends.local_id(); + slot.settled(Ok(())); + + if let Some(payload) = nwk_frame.payload.as_opaque() + && let Ok(ApsFrame::Data(aps_frame)) = parse_aps_frame(payload) + { + tracing::debug!( + "Delivering frame to ourselves locally: cluster {:#06x}", + aps_frame.cluster_id + ); + self.handle_zdp_frame(nwk_frame, &aps_frame); + self.push_notification(super::ZigbeeNotification::ReceivedApsCommand { + source: nwk_frame.nwk_header.source, + destination: nwk_frame.nwk_header.destination, + group: aps_frame.group_id, + profile_id: aps_frame.profile_id, + cluster_id: aps_frame.cluster_id, + src_ep: aps_frame.source_endpoint, + dst_ep: aps_frame.destination_endpoint.unwrap_or(0), + lqi: 255, + rssi: 0, + aps_encrypted: false, + data: aps_frame.asdu.to_vec(), + }); + } + SendHandle::new(id, slot) + } + /// Send a unicast: assign its NWK sequence number, resolve a next hop, and enqueue /// it, held for a route discovery if none is known. Resolution and enqueueing share /// one core lock, so the next hop cannot change in between. `Err` means nothing was @@ -358,6 +393,13 @@ impl ZigbeeStack { ); let destination = nwk_frame.nwk_header.destination; + + // SPIKE (#84 bench): a frame addressed to ourselves (zigpy scanning the + // coordinator's own tables with ZDP) is delivered locally instead of routed + if destination == self.state.network_address { + return Ok(self.deliver_locally(&nwk_frame)); + } + let now = self.core_now(); let mut core = self.core(); diff --git a/crates/ziggurat-driver/src/zigbee_stack/zdp.rs b/crates/ziggurat-driver/src/zigbee_stack/zdp.rs index 4775f22..0d22ba2 100644 --- a/crates/ziggurat-driver/src/zigbee_stack/zdp.rs +++ b/crates/ziggurat-driver/src/zigbee_stack/zdp.rs @@ -83,7 +83,10 @@ impl ZigbeeStack { extended_pan_id: self.state.extended_pan_id, extended_address: entry.extended_address, network_address: entry.network_address, + // Entries learned from Link Status default to Router; 0x0000 is + // always the coordinator (SPIKE #84 bench fix) device_type: match entry.device_type { + _ if entry.network_address == Nwk(0x0000) => ZdpDeviceType::Coordinator, NwkDeviceType::Coordinator => ZdpDeviceType::Coordinator, NwkDeviceType::Router => ZdpDeviceType::Router, NwkDeviceType::EndDevice => ZdpDeviceType::EndDevice, @@ -102,9 +105,10 @@ impl ZigbeeStack { _ => ZdpAffinity::NoneOfTheAbove, }, permit_joining: ZdpPermitJoining::Unknown, - // We are the coordinator (tree depth 0), so every neighbor of - // ours sits at depth 1 - depth: 1, + // The coordinator sits at tree depth 0, its neighbors at 1. A + // router does not track depth; its neighbors are reported at 1 + // like the coordinator's (SPIKE #84 bench fix for 0x0000) + depth: u8::from(entry.network_address != Nwk(0x0000)), lqa: entry.lqa().unwrap_or(0), }) .collect(); diff --git a/crates/ziggurat-phy-trel/BROADCAST.md b/crates/ziggurat-phy-trel/BROADCAST.md index c11f4ff..16dd43f 100644 --- a/crates/ziggurat-phy-trel/BROADCAST.md +++ b/crates/ziggurat-phy-trel/BROADCAST.md @@ -319,3 +319,174 @@ Optional suppression - Stop the coordinator for 60 s with two satellites and check that they stay neighbours. - Re-run the deny experiment (On/Off through the satellite) in each transport. + +# Secure ZLOE over QUIC: design and gap analysis + +Status: design (2026-11-10). Nothing built yet. + +Styleguide: as above (`.pre-commit-config.yaml`: `cargo fmt`, clippy with +`-D warnings -Dclippy::all -Dclippy::nursery`). New code lives in +`crates/ziggurat-phy-trel` (`security` module) with the server flags in +`crates/ziggurat-server`. + +## Goal + +Today the backbone is trusted: anyone on the LAN can read ZLOE headers, inject frames +and fetch the network key from the coordinator. Goal: authenticated and encrypted +ZLOE, without giving up statelessness (Ziggurat keeps no state between starts) and +without breaking the transports above (unicast, multicast, fallback). + +## Trust model + +- **Root of trust: the Zigbee network key.** Every ZLOE secret is derived from it + with HKDF-SHA256, salt = extended PAN ID: + - `ca_key = HKDF(nwk_key, "zloe ca v1")` → Ed25519 network CA (deterministic, so + every member can recompute it and nothing is stored), + - `mc_key = HKDF(nwk_key, "zloe multicast v1")` → ChaCha20-Poly1305 group key. +- **Authorization = network membership.** A node that holds the network key is a + backbone member, as it already is for Zigbee itself. Roles (coordinator / satellite) + are claims inside the node's certificate, signed by the network CA. +- **Revocation = network key rotation** (ZHA can rotate the network key): new key, new + CA, every old certificate stops verifying. +- **Enrollment** gives a new satellite the network key. It is the only step that needs + a secret from outside: a one-time enrollment code. + +## Protocol + +1. **Node identity.** At start a member generates an ephemeral Ed25519 key pair and + issues itself a certificate from the derived CA: subject = its EUI64, extension = + role and NWK address, validity = start time ± 1 year (clocks on MCUs are not + trusted; validity is not what revocation relies on). +2. **Peer links.** One QUIC connection (quinn + rustls, ALPN `zloe/1`) per peer pair: + mutual TLS, both certificates must chain to the network CA and name the EUI64 the + peer announces over mDNS. The lower EUI64 dials, the higher accepts, so a pair has + one connection. +3. **Frames over QUIC.** 802.15.4 frames travel as QUIC DATAGRAM frames (RFC 9221): + unreliable, no head-of-line blocking. ZLOE keeps its own per-frame ack (50 ms ×3), + carried as a datagram too. +4. **Link health from QUIC.** RTT and loss from the QUIC connection replace the + ack-success estimate; keepalive PINGs replace the multicast probe for unicast peers. +5. **Multicast.** Datagrams to the groups are sealed with `mc_key` + (ChaCha20-Poly1305, nonce = sender EUI64 ‖ 32-bit packet number, header as AAD). + Each sender also has a random 32-bit **epoch** per start, sent to every peer over + its QUIC connection; receivers accept multicast only for a sender's current epoch + and reject packet numbers already seen in it (replay window). The ZLOE header + gains the epoch field. +6. **Enrollment.** `ziggurat --zloe-satellite --zloe-enroll `: + - The coordinator prints a fresh code when asked (`zloe-ctl enroll`, later a ZHA + button): 128 random bits, base32, valid 10 minutes, one use. + - The satellite opens QUIC to the coordinator with ALPN `zloe-enroll/1`, accepting + any server certificate. + - Both sides compute `proof = HMAC(code, TLS exporter("zloe enroll", 32) ‖ role)` + and exchange proofs on a stream; each verifies the other's. A man in the middle + has two different TLS sessions and no code, so it cannot produce either proof; + with 128 bits it cannot brute-force the code from what it observes. + - On success the coordinator sends the network settings (the `NetworkInfoPayload` + the unauthenticated bootstrap sends today) and the connection closes. +7. **Modes.** `--zloe-security none|quic` (default `none` in the spike, so today's + behaviour stays available). Nodes announce their mode in mDNS TXT `sec`; a `quic` + node ignores `none` peers. + +## Gap analysis + +| ID | Gap | Severity | Resolution | +|---|---|---|---| +| Q1 | Microcontroller satellites: spinel2quic measured ~37 KB heap and ~641 ms per QUIC handshake on an ESP32 (ngtcp2). A full mesh means N−1 connections per node. | must for MCU satellites | Host satellites first. For MCUs: connect only to the coordinator plus peers it exchanges unicast with (lazy connections, closed when idle). | +| Q2 | quinn's default loss recovery stretches outages: PTO backoff turned a 3 s outage into a 5.4 s stall (spinel2quic). | must | Fixed congestion window, short max ack delay, 10 ms keepalive datagrams while traffic flows, as tuned in spinel2quic. | +| Q3 | Statelessness: the CA and multicast key are derived from the network key, so a leaked network key gives full backbone access. | accepted | Same trust as Zigbee itself (the network key already decrypts all NWK traffic). Rotation revokes everything. | +| Q4 | Network key rotation: nodes must re-derive keys and reconnect; peers on the old key stop verifying. | must | Rebuild identity and reconnect on `set_extended_pan_id`/network key change; old connections closed; rotation window where both keys are accepted (Zigbee keeps the previous key sequence too). | +| Q5 | Certificate validity needs time: MCUs boot with no clock. | minor | Long validity, time checks disabled in the verifier; certificates are ephemeral and re-issued every start. | +| Q6 | Group key for multicast lets any member forge another member's multicast. | accepted | Same as Zigbee NWK security (any key holder can forge NWK frames). Per-sender keys over QUIC are the upgrade if needed. | +| Q7 | Multicast replay after a sender restart: packet numbers restart randomly. | must | Per-start epoch announced over the authenticated QUIC link; receivers drop unknown epochs. | +| Q8 | Multicast from a sender we have no QUIC connection to yet (startup race) is dropped. | minor | Zigbee retries; the first Link Status round after the connection is up repairs it. | +| Q9 | Enrollment code UX: ZHA has no UI for it. | open | Spike: printed by the coordinator (`zloe-ctl enroll`). Later: ZHA repair/config flow, or a QR code. | +| Q10 | Enrollment code brute force. | covered | 128-bit codes, one use, 10 minute lifetime; proofs bound to the TLS session. A human-typeable short code would need a PAKE (SPAKE2) instead of HMAC. | +| Q11 | mDNS stays unauthenticated: an attacker can announce fake peers or addresses. | covered | Announcements only trigger connection attempts; a peer is accepted only after mutual TLS. Bad addresses cost retries (address rotation exists). | +| Q12 | Mixed networks (some nodes `none`, some `quic`). | minor | `quic` nodes ignore `none` peers; the coordinator decides which mode the network uses and satellites follow its TXT `sec`. | +| Q13 | Datagram size: 802.15.4 frame ≤127 B + ZLOE header + QUIC/UDP/IPv6 overhead. | covered | Well under the 1200 B minimum QUIC path MTU. | +| Q14 | Dependencies (quinn, rustls, ring/aws-lc, rcgen) for the host build; the `no_std` crates are untouched. | minor | Feature-gated in `ziggurat-phy-trel` and `ziggurat-server`. | +| Q15 | The unauthenticated bootstrap (`NETINFO_REQ`) still hands out the network key in `none` mode. | accepted | `quic` mode disables it; enrollment replaces it. | + +## Behaviour + +``` +Security (ziggurat-phy-trel::security) + derive_keys + Network key and extended PAN ID known + - derives the same CA key and multicast key on every node + - derives different keys for a different network key or extended PAN ID + issue_identity + Network keys derived + - creates a certificate for this node's EUI64 signed by the network CA + - includes role and NWK address +Backbone (QUIC mode) + connect_peer + Peer announced over mDNS with sec=quic, lower EUI64 is us + - dials the peer with ALPN zloe/1 and our certificate + Peer's certificate chains to the network CA and names the announced EUI64 + - accepts the connection + - exchanges multicast epochs + Peer's certificate is signed by another CA (other network key) + - closes the connection + - logs the rejection once + Peer's certificate names a different EUI64 than announced + - closes the connection + Peer announces sec=none + - does not connect + send_unicast + Peer connected + - sends the frame as a QUIC datagram + - returns Acked when the ZLOE ack datagram arrives + No ack after 3 tries + - falls back to the radio + Peer not connected + - falls back to the radio + send_broadcast + Peers reachable by multicast + - sends one datagram per group, sealed with the multicast key + Other peers + - sends a QUIC datagram copy to each + receive_multicast + Sealed with the multicast key, sender's current epoch, new packet number + - delivers the frame once + Fails authentication + - drops it + Unknown epoch for that sender + - drops it + Packet number already seen in the epoch + - drops it + rotate_network_key + Network key changes + - derives new keys and certificates + - closes connections made with the old identity + - reconnects to peers +Enrollment + coordinator issue_code + Requested by zloe-ctl enroll + - returns a 128-bit base32 code valid 10 minutes + satellite enroll + Correct code + - both proofs verify + - receives the network settings + - starts in router role in quic mode + Wrong code + - proofs fail on both sides + - receives nothing + - coordinator logs the failed attempt + Code expired or already used + - coordinator rejects the proof + Man in the middle without the code + - neither side's proof verifies +``` + +## Spike plan + +1. Key derivation + self-issued certificates (rcgen), verifier pinned to the derived + CA. +2. QUIC peer links (quinn) carrying datagrams, with the spinel2quic tuning; ZLOE ack + over datagrams; fall back to the radio when not connected. +3. Sealed multicast with epochs. +4. Enrollment with code and exporter-bound proofs; `none` mode bootstrap disabled + in `quic` mode. +5. Bench: the deny run, satellite bootstrap via enrollment, a rogue node with another + network key rejected, multicast replay rejected, coordinator restart. diff --git a/crates/ziggurat-zigbee/src/sends/mod.rs b/crates/ziggurat-zigbee/src/sends/mod.rs index 77a5c12..26d97f8 100644 --- a/crates/ziggurat-zigbee/src/sends/mod.rs +++ b/crates/ziggurat-zigbee/src/sends/mod.rs @@ -266,6 +266,12 @@ impl Default for Sends { } impl> Sends { + /// An id for a send that never enters the table (delivered locally, e.g. a frame + /// we address to ourselves). + pub const fn local_id(&mut self) -> SendId { + self.next_id() + } + const fn next_id(&mut self) -> SendId { let id = SendId(self.next_id); self.next_id += 1; From 0b377ddbeb00eb0f8ccfd8cfa18c8a78d8c6344e Mon Sep 17 00:00:00 2001 From: Igor Yamolov Date: Tue, 10 Nov 2026 18:04:26 +0100 Subject: [PATCH 18/22] SPIKE: secure ZLOE over QUIC with enrollment (PoC) --zloe-security quic: peer links over QUIC with mutual TLS against a network CA derived from the network key (stateless, revoked by key rotation), ZLOE datagrams as QUIC DATAGRAM frames, broadcasts as one datagram per peer. Plain UDP data, multicast and the unauthenticated settings hand-out are off in this mode. Satellites enroll over QUIC with a one-time 128-bit code, proofs bound to the TLS session (--zloe-enroll ), or with --zloe-enroll auto while the coordinator is in pairing mode (permit-join open). Loopback tests: secure exchange, other network key refused, code enrollment (once, wrong code refused), auto-enrollment gated on pairing. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 251 ++++++++- .../src/zigbee_stack/joining.rs | 2 +- crates/ziggurat-phy-trel/BROADCAST.md | 37 ++ crates/ziggurat-phy-trel/Cargo.toml | 5 + crates/ziggurat-phy-trel/src/lib.rs | 505 +++++++++++++----- crates/ziggurat-phy-trel/src/quic.rs | 411 ++++++++++++++ crates/ziggurat-phy-trel/src/security.rs | 365 +++++++++++++ crates/ziggurat-phy-trel/tests/loopback.rs | 1 + crates/ziggurat-phy-trel/tests/quic.rs | 290 ++++++++++ crates/ziggurat-server/src/main.rs | 59 +- crates/ziggurat-server/src/satellite.rs | 30 ++ 11 files changed, 1828 insertions(+), 128 deletions(-) create mode 100644 crates/ziggurat-phy-trel/src/quic.rs create mode 100644 crates/ziggurat-phy-trel/src/security.rs create mode 100644 crates/ziggurat-phy-trel/tests/quic.rs diff --git a/Cargo.lock b/Cargo.lock index 0561fed..75fc008 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -131,6 +131,12 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "bitflags" version = "1.3.2" @@ -447,6 +453,12 @@ version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" +[[package]] +name = "deranged" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e9de72ce2ad1f90dc62fa25f0f430ef85eb4b0d8fa0be4f30373bc40a21d28e" + [[package]] name = "derive-into-owned" version = "0.2.0" @@ -748,6 +760,19 @@ dependencies = [ "windows-result", ] +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + [[package]] name = "getrandom" version = "0.4.3" @@ -755,9 +780,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi", "rand_core", + "wasm-bindgen", ] [[package]] @@ -966,6 +993,12 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "lru-slab" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4050469837a6ff301cd14c1f8f24f88549e6d548f24f64e2148eb0f72cebc51f" + [[package]] name = "mach2" version = "0.4.3" @@ -1077,6 +1110,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + [[package]] name = "num-traits" version = "0.2.19" @@ -1154,12 +1193,28 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "pem" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" +dependencies = [ + "base64", + "serde_core", +] + [[package]] name = "pin-project-lite" version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +[[package]] +name = "powerfmt" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a6394b9e965e73d0a289ee54f589087e2c676aedf60885baf52c76b771e4958" + [[package]] name = "proc-macro-crate" version = "3.5.0" @@ -1200,6 +1255,62 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "quinn" +version = "0.11.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4051e23e9185c255a7e33ef59cdbca87a22d359052eecd22fc6b901fb37d9d11" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.21", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e750cca55fe4f0439a15d0bb529da9651e79993e8e72c61a899a36d462befbe" +dependencies = [ + "bytes", + "getrandom 0.4.3", + "lru-slab", + "rand", + "rand_pcg", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.21", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af66907df18639dcf4db56ca65490cabc4b27a97dbadd96f2926cca73298f016" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.61.2", +] + [[package]] name = "quote" version = "1.0.47" @@ -1228,7 +1339,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af" dependencies = [ "chacha20", - "getrandom", + "getrandom 0.4.3", "rand_core", ] @@ -1238,6 +1349,28 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core", +] + +[[package]] +name = "rcgen" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75e669e5202259b5314d1ea5397316ad400819437857b90861765f24c4cf80a2" +dependencies = [ + "pem", + "ring", + "rustls-pki-types", + "time", + "yasna", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -1264,6 +1397,61 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustls" +version = "0.23.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + [[package]] name = "rustversion" version = "1.0.23" @@ -1500,6 +1688,31 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "time" +version = "0.3.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" +dependencies = [ + "deranged", + "num-conv", + "powerfmt", + "serde_core", + "time-core", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "tinyvec" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" + [[package]] name = "tokio" version = "1.53.1" @@ -1682,6 +1895,12 @@ version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + [[package]] name = "utf8parse" version = "0.2.2" @@ -1751,6 +1970,16 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "windows-core" version = "0.62.2" @@ -1910,6 +2139,21 @@ dependencies = [ "tap", ] +[[package]] +name = "yasna" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd" +dependencies = [ + "time", +] + +[[package]] +name = "zeroize" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13084392c5e4bc371903e2935a5eaeed24905a7511356b883835e18a78f6879" + [[package]] name = "ziggurat-driver" version = "0.1.0" @@ -1970,6 +2214,11 @@ dependencies = [ "if-addrs", "mdns-sd", "parking_lot", + "quinn", + "rcgen", + "ring", + "rustls", + "rustls-webpki", "socket2", "tokio", "tracing", diff --git a/crates/ziggurat-driver/src/zigbee_stack/joining.rs b/crates/ziggurat-driver/src/zigbee_stack/joining.rs index 19ef6e0..e344e46 100644 --- a/crates/ziggurat-driver/src/zigbee_stack/joining.rs +++ b/crates/ziggurat-driver/src/zigbee_stack/joining.rs @@ -1422,7 +1422,7 @@ impl ZigbeeStack { } /// Whether the trust center authorizes new joins through a router right now. - pub(super) fn trust_center_permitting_joins(&self) -> bool { + pub fn trust_center_permitting_joins(&self) -> bool { self.core() .trust_center_joins_until .is_some_and(|deadline| deadline > self.core_now()) diff --git a/crates/ziggurat-phy-trel/BROADCAST.md b/crates/ziggurat-phy-trel/BROADCAST.md index 16dd43f..661f9e8 100644 --- a/crates/ziggurat-phy-trel/BROADCAST.md +++ b/crates/ziggurat-phy-trel/BROADCAST.md @@ -490,3 +490,40 @@ Enrollment in `quic` mode. 5. Bench: the deny run, satellite bootstrap via enrollment, a rogue node with another network key rejected, multicast replay rejected, coordinator restart. + +## QUIC PoC as built (2026-11-10) + +`--zloe-security quic` (default `none`): + +- `security.rs`: HKDF-SHA256 from the network key (salt = extended PAN ID) to an + Ed25519 seed for the network CA; the CA certificate comes out byte-identical on every + member. Each node issues itself an ephemeral Ed25519 certificate for + `x.zloe`, validity fixed 2000–2100. `mcast_key` is derived but unused (no + multicast in secure mode yet). +- `quic.rs`: one quinn endpoint per node (dual-stack, `--zloe-quic-port`, default + ZLOE port + 1), transport tuned as in spinel2quic (fixed window, immediate acks, + 10 ms initial RTT, 1 s keepalive, 10 s idle timeout). ALPN `zloe/1`: mutual TLS + against the network CA, the dialer sends its EUI64 on a uni stream and the acceptor + checks the certificate names it. ZLOE datagrams (same header) travel as QUIC DATAGRAM + frames with the usual ack/retry; broadcasts go as one datagram per connected peer. +- Secure mode switches off plain UDP data, multicast and the unauthenticated + `NETINFO_REQ` hand-out. mDNS TXT gains `sec=quic|none` and `qp`; nodes only peer + with nodes of the same mode. +- Enrollment (ALPN `zloe-enroll/1`): proofs `HMAC(code, exporter ‖ side)` exchanged on + a bi stream; the coordinator then sends the `NetworkInfoPayload`. Codes are 128-bit + base32, one use, 10 minutes, logged and written to `--zloe-enroll-code-file`. +- **Auto-enrollment:** while the coordinator's trust center permits joins (ZHA pairing + mode), it also accepts the empty code. `--zloe-satellite --zloe-enroll auto`. + +Loopback tests (`tests/quic.rs`, fake radios): secure peers exchange an acked unicast +over QUIC; a node with another network key is discovered but never gets a link; +enrollment with the code works once and refuses a wrong code; auto-enrollment is +refused while pairing is closed and works once it opens. 5/5 runs green. + +### Auto-enrollment gaps + +| ID | Gap | Severity | Resolution | +|---|---|---|---| +| A1 | During the pairing window anyone on the LAN can enroll and get the network key. | accepted | Same exposure as Zigbee joining with the well-known link key during permit-join; window is ZHA's (default 254 s). Code enrollment for when that is not acceptable. | +| A2 | The satellite cannot authenticate the coordinator (empty code): a LAN attacker answering first can hand it a fake network. | accepted for the spike | Pin the coordinator's CA fingerprint shown in ZHA, or confirm the satellite in ZHA before it gets the key (two-step approval). | +| A3 | No record of which satellites enrolled. | minor | Coordinator logs each enrollment; ZHA lists satellites after their Device_annce. | diff --git a/crates/ziggurat-phy-trel/Cargo.toml b/crates/ziggurat-phy-trel/Cargo.toml index 18d7adc..2e69ff4 100644 --- a/crates/ziggurat-phy-trel/Cargo.toml +++ b/crates/ziggurat-phy-trel/Cargo.toml @@ -18,6 +18,11 @@ tracing = "0.1" mdns-sd = "0.21.7" if-addrs = "0.15" socket2 = { version = "0.6.5", features = ["all"] } +quinn = { version = "0.11", default-features = false, features = ["runtime-tokio", "rustls-ring"] } +rustls = { version = "0.23", default-features = false, features = ["ring", "std"] } +ring = "0.17" +rcgen = "0.13" +rustls-webpki = { version = "0.103.16", default-features = false, features = ["ring", "std"] } [dev-dependencies] tokio = { version = "1.53.1", features = ["rt-multi-thread", "macros", "time", "sync", "net"] } diff --git a/crates/ziggurat-phy-trel/src/lib.rs b/crates/ziggurat-phy-trel/src/lib.rs index 762b8e7..88d2f57 100644 --- a/crates/ziggurat-phy-trel/src/lib.rs +++ b/crates/ziggurat-phy-trel/src/lib.rs @@ -24,6 +24,9 @@ //! report `NoAck` without being transmitted. The RCP still auto-ACKs frames from denied //! devices at the hardware level; see the spike notes. +pub mod quic; +pub mod security; + use std::collections::{HashMap, HashSet, VecDeque}; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, SocketAddrV4, SocketAddrV6}; use std::sync::Arc; @@ -106,9 +109,20 @@ pub struct TrelConfig { pub mcast_block_file: Option, /// Test hook: behave like a host without IPv6 (IPv4 socket, no IPv6 peers or group) pub ipv4_only: bool, + /// Secure mode: peer links over authenticated QUIC; plain UDP data, multicast and + /// the unauthenticated settings hand-out are off + pub quic: Option, pub deny: DenyList, } +#[derive(Debug, Clone)] +pub struct QuicConfig { + /// UDP port of the QUIC endpoint + pub port: u16, + /// Coordinator: where to write the current enrollment code + pub code_file: Option, +} + /// The addresses of the interface the backbone is confined to. #[derive(Debug, Clone, Copy)] struct LocalNet { @@ -153,6 +167,8 @@ struct Peer { /// The multicast group and port the peer listens on (mDNS TXT `mc`), scoped to /// our backbone interface. `None`: the peer cannot be reached by multicast. mcast: [Group; 2], + /// QUIC port the peer announced (TXT `qp`), secure mode only + quic_port: Option, /// The confirmation state last logged, to log each change once mcast_logged_ok: bool, } @@ -167,6 +183,7 @@ impl Peer { health: 1.0, alternates: Vec::new(), mcast: [Group::default(), Group::default()], + quic_port: None, mcast_logged_ok: false, } } @@ -263,6 +280,10 @@ struct Shared { coordinators: HashMap)>, /// Everything mDNS resolved, by extended address resolved: HashMap, + /// Secure (QUIC) mode + secure: bool, + /// QUIC ports of coordinators seen on the backbone (enrollment targets) + coordinator_quic: HashMap, netinfo_provider: Option, netinfo_waiter: Option>>, /// Packet numbers of our latest multicast probes, with their family @@ -300,6 +321,8 @@ struct Backbone { /// The task reading our multicast socket; replaced when the network changes mcast_reader: Mutex>>, mcast4_reader: Mutex>>, + /// Secure mode: the QUIC link to peers + quic: Option>, } pub struct TrelPhy { @@ -369,10 +392,42 @@ impl TrelPhy

{ None }; + // Secure mode: QUIC peer links; its datagrams go through the same handler + let quic = match &config.quic { + Some(qc) => { + let (h_shared, h_pending) = + (Arc::clone(&shared), Arc::new(Mutex::new(HashMap::new()))); + let handler: quic::Handler = { + let h_pending: Arc>>> = + Arc::clone(&h_pending); + let h_shared = Arc::clone(&h_shared); + Arc::new(move |buf: &[u8], from: SocketAddr| { + process_datagram(&h_shared, &h_pending, buf, from, false) + }) + }; + let n_shared = Arc::clone(&shared); + let netinfo: quic::NetInfo = Arc::new(move || { + let provider = n_shared.lock().netinfo_provider.clone(); + provider.and_then(|provide| provide()) + }); + shared.lock().secure = true; + Some(( + quic::QuicLink::new(qc.port, handler, netinfo, qc.code_file.clone())?, + h_pending, + )) + } + None => None, + }; + let (quic, pending) = match quic { + Some((link, pending)) => (Some(link), pending), + None => (None, Arc::new(Mutex::new(HashMap::new()))), + }; + let backbone = Backbone { + quic, socket, shared: Arc::clone(&shared), - pending: Arc::new(Mutex::new(HashMap::new())), + pending, seq: Arc::new(AtomicU32::new(rand_u32())), stats: Arc::new(Stats::default()), mcast_block_file: config.mcast_block_file, @@ -382,7 +437,11 @@ impl TrelPhy

{ local_net, prefer_ipv4: config.prefer_ipv4, dual_stack, - mcast_port: config.mcast_port, + mcast_port: if config.quic.is_some() { + 0 + } else { + config.mcast_port + }, mcast_reader: Mutex::new(None), mcast4_reader: Mutex::new(None), }; @@ -417,6 +476,79 @@ impl TrelPhy

{ } } + /// Secure mode: the network key is known (configured, or after enrollment). Derives + /// the network CA and our certificate and starts the QUIC links. + pub fn set_network_key( + &self, + nwk_key: &[u8; 16], + epan: Eui64, + me: Eui64, + coordinator: bool, + ) -> std::io::Result<()> { + let Some(link) = self.backbone.as_ref().and_then(|b| b.quic.as_ref()) else { + return Ok(()); + }; + let role = if coordinator { + security::Role::Coordinator + } else { + security::Role::Satellite + }; + link.set_network(nwk_key, epan, me, role) + } + + /// Secure mode, coordinator side: whether the network is in pairing mode, when + /// satellites may enroll without a code. + pub fn set_pairing_open(&self, pairing_open: quic::PairingOpen) { + if let Some(link) = self.backbone.as_ref().and_then(|b| b.quic.as_ref()) { + link.set_pairing_open(pairing_open); + } + } + + /// Secure mode, satellite side: find a coordinator on the backbone (on `epan`, when + /// given) and enroll with `code`: returns the network settings it sends. + pub async fn enroll( + &self, + epan: Option, + code: &str, + timeout: Duration, + ) -> std::io::Result> { + let Some(backbone) = &self.backbone else { + return Err(std::io::Error::other("ZLOE backbone is disabled")); + }; + let Some(link) = &backbone.quic else { + return Err(std::io::Error::other( + "enrollment needs --zloe-security quic", + )); + }; + let deadline = tokio::time::Instant::now() + timeout; + loop { + let target = { + let shared = self.shared.lock(); + shared + .coordinators + .iter() + .filter(|(_, (_, e))| epan.is_none() || *e == epan) + .find_map(|(eui64, (addr, _))| { + shared + .coordinator_quic + .get(eui64) + .map(|qp| (*eui64, SocketAddr::new(addr.ip(), *qp))) + }) + }; + if let Some((eui64, addr)) = target { + tracing::info!("ZLOE/QUIC: enrolling with coordinator {eui64:?} at {addr}"); + return link.enroll(addr, code).await; + } + if tokio::time::Instant::now() >= deadline { + return Err(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "no secure coordinator on the ZLOE backbone", + )); + } + tokio::time::sleep(Duration::from_millis(250)).await; + } + } + /// Coordinator side: hand these settings to satellites that ask over the backbone. pub fn set_netinfo_provider(&self, provider: NetInfoProvider) { self.shared.lock().netinfo_provider = Some(provider); @@ -477,6 +609,138 @@ impl TrelPhy

{ } } +/// Handle one ZLOE datagram (from UDP or a QUIC link); returns the reply to send back +/// to `from` (an ack or the network settings), if any. +fn process_datagram( + shared: &Mutex, + pending: &Mutex>>, + buf: &[u8], + from: SocketAddr, + allow_netinfo: bool, +) -> Option> { + let Some(header) = WireHeader::parse(buf) else { + tracing::debug!("ZLOE: dropping malformed datagram from {from}"); + return None; + }; + + if header.kind == KIND_ACK { + let waiter = pending.lock().remove(&header.seq); + if let Some(waiter) = waiter { + let _ = waiter.send(()); + } else { + // Not a unicast we are waiting on: an answer to a probe? + let mut shared = shared.lock(); + let family = shared + .probes + .iter() + .find(|(seq, _)| *seq == header.seq) + .map(|(_, f)| *f); + if let Some(family) = family + && let Some(peer) = shared.peers.get_mut(&header.eui64) + { + peer.mcast[family].acked = Some(Instant::now()); + } + } + return None; + } + + // Bootstrap exchange: the backbone is trusted, so the settings (network + // key included) go to whoever asks + if header.kind == KIND_NETINFO_REQ && allow_netinfo { + let (provider, identity) = { + let shared = shared.lock(); + (shared.netinfo_provider.clone(), shared.identity) + }; + let info = provider.and_then(|provide| provide())?; + tracing::info!( + "ZLOE: sending network settings to {from} ({:?})", + header.eui64 + ); + let mut rsp = netinfo_header(KIND_NETINFO_RSP, header.seq, identity); + rsp.extend_from_slice(&info); + return Some(rsp); + } + if header.kind == KIND_NETINFO_RSP { + if let Some(waiter) = shared.lock().netinfo_waiter.take() { + let _ = waiter.send(buf[HEADER_LEN..].to_vec()); + } + return None; + } + + if header.kind != KIND_DATA && header.kind != KIND_PROBE { + return None; + } + let psdu = buf[HEADER_LEN..].to_vec(); + let (deliver, ack_to) = { + let mut shared = shared.lock(); + let identity = shared.identity?; + if header.epan != identity.epan || header.eui64 == identity.eui64 { + return None; + } + + // Learn (or refresh) the peer from its own traffic + let peer = shared.peers.entry(header.eui64).or_insert_with(|| { + tracing::info!( + "ZLOE: learned peer {:?} / {:?} at {from}", + header.eui64, + header.nwk + ); + Peer::new(from, None, PeerSource::Learned) + }); + peer.addr = from; + peer.nwk = Some(header.nwk); + peer.last_rx = Some(Instant::now()); + let lqi = peer.lqi(); + + // A probe only refreshes the peer and gets its ack + let seen = shared.seen.entry(header.eui64).or_default(); + let duplicate = header.kind == KIND_PROBE || seen.contains(&header.seq); + if !duplicate { + seen.push_back(header.seq); + if seen.len() > DEDUP_WINDOW { + seen.pop_front(); + } + } + + let ack_to = (header.flags & FLAG_ACK_REQUEST != 0).then_some(from); + let deliver = (!duplicate).then(|| { + ( + shared.rx_tx.clone(), + RxFrame { + psdu, + channel: identity.channel, + rssi: BACKBONE_RSSI, + lqi, + timestamp_us: 0, + }, + ) + }); + (deliver, ack_to.map(|to| (to, identity))) + }; + + if let Some((Some(tx), frame)) = deliver { + tracing::trace!( + "ZLOE: rx {} bytes from {:?}", + frame.psdu.len(), + header.eui64 + ); + let _ = tx.try_send(frame); + } + if let Some((_, identity)) = ack_to { + let ack = WireHeader { + kind: KIND_ACK, + flags: 0, + seq: header.seq, + eui64: identity.eui64, + nwk: identity.nwk, + epan: identity.epan, + } + .to_bytes(); + return Some(ack); + } + None +} + /// Header for the bootstrap messages, sent before we may have an identity. fn netinfo_header(kind: u8, seq: u32, identity: Option) -> Vec { WireHeader { @@ -499,6 +763,8 @@ struct Resolved { candidates: Vec<((u8, u8), SocketAddr)>, /// Announced groups: [IPv6 (`mc`), IPv4 (`mc4`)] mcast: [Option; 2], + /// Security mode (`sec`) and QUIC port (`qp`) + quic_port: Option, } /// Create or update the peer for a resolved node on our network. @@ -512,6 +778,10 @@ fn apply_resolved(shared: &mut Shared, eui64: Eui64) { if eui64 == identity.eui64 || resolved.epan != Some(identity.epan) { return; } + // A secure node only peers with secure nodes, and the other way around + if shared.secure != resolved.quic_port.is_some() { + return; + } let Some(&(best_key, addr)) = resolved.candidates.first() else { return; }; @@ -530,6 +800,7 @@ fn apply_resolved(shared: &mut Shared, eui64: Eui64) { .find(|(_, a)| *a == peer.addr) .map(|(k, _)| *k); peer.alternates = resolved.candidates.iter().map(|(_, a)| *a).collect(); + peer.quic_port = resolved.quic_port; for family in [V6, V4] { if peer.mcast[family].addr != resolved.mcast[family] { tracing::info!( @@ -624,6 +895,7 @@ impl Backbone { let shared = Arc::clone(&self.shared); let pending = Arc::clone(&self.pending); let dual_stack = self.dual_stack; + let secure = self.quic.is_some(); tokio::spawn(async move { let mut buf = vec![0u8; 2048]; @@ -643,131 +915,12 @@ impl Backbone { { continue; } - let Some(header) = WireHeader::parse(&buf[..len]) else { - tracing::debug!("ZLOE: dropping malformed datagram from {from}"); - continue; - }; - - if header.kind == KIND_ACK { - let waiter = pending.lock().remove(&header.seq); - if let Some(waiter) = waiter { - let _ = waiter.send(()); - } else { - // Not a unicast we are waiting on: an answer to a probe? - let mut shared = shared.lock(); - let family = shared - .probes - .iter() - .find(|(seq, _)| *seq == header.seq) - .map(|(_, f)| *f); - if let Some(family) = family - && let Some(peer) = shared.peers.get_mut(&header.eui64) - { - peer.mcast[family].acked = Some(Instant::now()); - } - } + // Secure mode: plain UDP carries nothing we trust + if secure { continue; } - - // Bootstrap exchange: the backbone is trusted, so the settings (network - // key included) go to whoever asks - if header.kind == KIND_NETINFO_REQ { - let (provider, identity) = { - let shared = shared.lock(); - (shared.netinfo_provider.clone(), shared.identity) - }; - let Some(info) = provider.and_then(|provide| provide()) else { - continue; - }; - tracing::info!( - "ZLOE: sending network settings to {from} ({:?})", - header.eui64 - ); - let mut rsp = netinfo_header(KIND_NETINFO_RSP, header.seq, identity); - rsp.extend_from_slice(&info); - let _ = socket.send_to(&rsp, to_family(from, dual_stack)).await; - continue; - } - if header.kind == KIND_NETINFO_RSP { - if let Some(waiter) = shared.lock().netinfo_waiter.take() { - let _ = waiter.send(buf[HEADER_LEN..len].to_vec()); - } - continue; - } - - if header.kind != KIND_DATA && header.kind != KIND_PROBE { - continue; - } - let psdu = buf[HEADER_LEN..len].to_vec(); - let (deliver, ack_to) = { - let mut shared = shared.lock(); - let Some(identity) = shared.identity else { - continue; - }; - if header.epan != identity.epan || header.eui64 == identity.eui64 { - continue; - } - - // Learn (or refresh) the peer from its own traffic - let peer = shared.peers.entry(header.eui64).or_insert_with(|| { - tracing::info!( - "ZLOE: learned peer {:?} / {:?} at {from}", - header.eui64, - header.nwk - ); - Peer::new(from, None, PeerSource::Learned) - }); - peer.addr = from; - peer.nwk = Some(header.nwk); - peer.last_rx = Some(Instant::now()); - let lqi = peer.lqi(); - - // A probe only refreshes the peer and gets its ack - let seen = shared.seen.entry(header.eui64).or_default(); - let duplicate = header.kind == KIND_PROBE || seen.contains(&header.seq); - if !duplicate { - seen.push_back(header.seq); - if seen.len() > DEDUP_WINDOW { - seen.pop_front(); - } - } - - let ack_to = (header.flags & FLAG_ACK_REQUEST != 0).then_some(from); - let deliver = (!duplicate).then(|| { - ( - shared.rx_tx.clone(), - RxFrame { - psdu, - channel: identity.channel, - rssi: BACKBONE_RSSI, - lqi, - timestamp_us: 0, - }, - ) - }); - (deliver, ack_to.map(|to| (to, identity))) - }; - - if let Some((to, identity)) = ack_to { - let ack = WireHeader { - kind: KIND_ACK, - flags: 0, - seq: header.seq, - eui64: identity.eui64, - nwk: identity.nwk, - epan: identity.epan, - } - .to_bytes(); - let _ = socket.send_to(&ack, to_family(to, dual_stack)).await; - } - - if let Some((Some(tx), frame)) = deliver { - tracing::trace!( - "ZLOE: rx {} bytes from {:?}", - frame.psdu.len(), - header.eui64 - ); - let _ = tx.try_send(frame); + if let Some(reply) = process_datagram(&shared, &pending, &buf[..len], from, true) { + let _ = socket.send_to(&reply, to_family(from, dual_stack)).await; } } }) @@ -822,6 +975,10 @@ impl Backbone { .filter(|_| local_net.is_some_and(|n| n.v4.is_some())) .map(SocketAddr::V4); let mcast = [mc6, mc4]; + let quic_port = (info.get_property_val_str("sec") == Some("quic")) + .then(|| info.get_property_val_str("qp")) + .flatten() + .and_then(|p| p.parse::().ok()); // mDNS lists every address of every interface (Docker bridges, // loopback, ...): pick deterministically let candidates = candidate_addresses( @@ -836,6 +993,11 @@ impl Backbone { }; let mut shared = shared.lock(); + if info.get_property_val_str("role") == Some("coordinator") + && let Some(qp) = quic_port + { + shared.coordinator_quic.insert(eui64, qp); + } if info.get_property_val_str("role") == Some("coordinator") { let known = shared.coordinators.insert(eui64, (addr, epan)); if known.is_none() { @@ -853,6 +1015,7 @@ impl Backbone { nwk, candidates, mcast, + quic_port, }, ); apply_resolved(&mut shared, eui64); @@ -1013,6 +1176,16 @@ impl Backbone { if let [_, Some(mc4)] = mcast { properties.insert("mc4".to_string(), mc4.to_string()); } + // Security mode; a secure node also announces its QUIC port + match &self.quic { + Some(link) => { + properties.insert("sec".to_string(), "quic".to_string()); + properties.insert("qp".to_string(), link.port.to_string()); + } + None => { + properties.insert("sec".to_string(), "none".to_string()); + } + } // Addresses (IPv6 link-local and IPv4) come from the interfaces mDNS runs on, // which --zloe-interface already narrows to one @@ -1042,11 +1215,29 @@ impl Backbone { let stats = Arc::clone(&self.stats); let prefer_ipv4 = self.prefer_ipv4; let dual_stack = self.dual_stack; + let quic = self.quic.clone(); tokio::spawn(async move { let mut tick = tokio::time::interval(PROBE_INTERVAL); loop { tick.tick().await; + // Secure mode: keep a QUIC link to every secure peer (the lower EUI64 + // dials); no multicast to probe + if let Some(link) = &quic { + let targets: Vec<(Eui64, SocketAddr)> = shared + .lock() + .peers + .iter() + .filter_map(|(eui64, p)| { + p.quic_port + .map(|qp| (*eui64, SocketAddr::new(p.addr.ip(), qp))) + }) + .collect(); + for (peer, addr) in targets { + link.ensure_connected(peer, addr); + } + continue; + } let probes = { let mut shared = shared.lock(); let Some(identity) = shared.identity else { @@ -1166,6 +1357,17 @@ impl Backbone { }; datagram.extend_from_slice(psdu); + // Secure mode: one QUIC datagram per connected peer + if let Some(link) = &self.quic { + self.stats.broadcasts.fetch_add(1, Ordering::Relaxed); + for peer in link.connected() { + if link.send(peer, &datagram) { + self.stats.unicast_copies.fetch_add(1, Ordering::Relaxed); + } + } + return; + } + // One multicast datagram covers every peer it is known to reach; the others // (no announced address, or probes unanswered) get a unicast copy with the // same packet number, so a peer that receives both keeps one @@ -1203,6 +1405,48 @@ impl Backbone { } } + /// Secure mode: unicast over the peer's QUIC link, acked like the UDP path. + async fn send_unicast_quic(&self, peer: Eui64, psdu: &[u8], ack_request: bool) -> bool { + let Some(link) = &self.quic else { return false }; + if !link.is_connected(peer) { + return false; + } + let seq = self.seq.fetch_add(1, Ordering::Relaxed); + let flags = if ack_request { FLAG_ACK_REQUEST } else { 0 }; + let Some(mut datagram) = self.header(KIND_DATA, flags, seq) else { + return false; + }; + datagram.extend_from_slice(psdu); + if !ack_request { + return link.send(peer, &datagram); + } + for _ in 0..ATTEMPTS { + let (tx, rx) = oneshot::channel(); + self.pending.lock().insert(seq, tx); + if !link.send(peer, &datagram) { + break; + } + if tokio::time::timeout(ACK_TIMEOUT, rx).await.is_ok() { + return true; + } + } + self.pending.lock().remove(&seq); + false + } + + fn peer_eui_for(&self, dest: &MacAddress) -> Option { + let shared = self.shared.lock(); + match dest { + MacAddress::Extended(eui64) => shared.peers.contains_key(eui64).then_some(*eui64), + MacAddress::Short(nwk) => shared + .peers + .iter() + .find(|(_, p)| p.nwk == Some(*nwk)) + .map(|(e, _)| *e), + MacAddress::None => None, + } + } + /// Reliable-ish unicast to one peer. `true` once acked. async fn send_unicast(&self, to: SocketAddr, psdu: &[u8], ack_request: bool) -> bool { let seq = self.seq.fetch_add(1, Ordering::Relaxed); @@ -1292,6 +1536,17 @@ impl RadioPhy for TrelPhy

{ if mac.frame_type == MAC_FRAME_DATA { backbone.send_broadcast(&frame.psdu).await; } + } else if backbone.quic.is_some() { + if let Some(peer) = backbone.peer_eui_for(&mac.dest) { + if backbone + .send_unicast_quic(peer, &frame.psdu, mac.ack_request) + .await + { + tracing::trace!("ZLOE: tx to {:?} via QUIC", mac.dest); + return Ok(TxResult::Acked); + } + tracing::info!("ZLOE: QUIC to {:?} failed, falling back to air", mac.dest); + } } else if let Some(peer) = backbone.peer_for(&mac.dest) { if backbone .send_unicast(peer, &frame.psdu, mac.ack_request) diff --git a/crates/ziggurat-phy-trel/src/quic.rs b/crates/ziggurat-phy-trel/src/quic.rs new file mode 100644 index 0000000..2acc0cf --- /dev/null +++ b/crates/ziggurat-phy-trel/src/quic.rs @@ -0,0 +1,411 @@ +//! SPIKE: secure ZLOE peer links over QUIC (see BROADCAST.md, "Secure ZLOE over QUIC"). +//! +//! One QUIC connection per peer pair (the lower EUI64 dials), mutual TLS against the +//! network CA derived from the network key, ZLOE datagrams carried as QUIC DATAGRAM +//! frames. The same endpoint serves enrollment (`zloe-enroll/1`) on the coordinator. + +use std::collections::{HashMap, HashSet}; +use std::net::{Ipv6Addr, SocketAddr}; +use std::path::PathBuf; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use parking_lot::Mutex; +use quinn::{AckFrequencyConfig, IdleTimeout, TransportConfig, VarInt}; +use rustls::pki_types::CertificateDer; +use ziggurat_ieee_802154::types::Eui64; + +use crate::security::{ + self, ALPN_ENROLL, ALPN_ZLOE, NetworkSecrets, Role, cert_names, enrollment_proof, exporter, + proof_matches, +}; + +/// Handles one incoming ZLOE datagram; returns the reply to send back, if any. +pub type Handler = Arc Option> + Send + Sync>; +/// The encoded network settings handed to an enrolling satellite. +pub type NetInfo = Arc Option> + Send + Sync>; +/// Whether the network is in pairing mode (permit-join open): enrollment without a +/// code is accepted then, like Zigbee joining with the well-known link key. +pub type PairingOpen = Arc bool + Send + Sync>; + +/// The code an auto-enrolling satellite proves: none. +pub const AUTO_CODE: &str = ""; + +const ENROLL_CODE_LIFETIME: Duration = Duration::from_secs(600); + +pub struct QuicLink { + endpoint: quinn::Endpoint, + pub port: u16, + state: Mutex, + handler: Handler, + netinfo: NetInfo, + code_file: Option, + pairing_open: Mutex>, +} + +#[derive(Default)] +struct State { + client: Option, + me: Option, + conns: HashMap, + dialing: HashSet, + /// Coordinator: the current enrollment code and when it was issued + code: Option<(String, Instant)>, + accepting: bool, +} + +/// spinel2quic's tuning: fixed congestion window (no backoff stalls), immediate acks. +fn transport() -> Arc { + #[derive(Clone)] + struct FixedWindow(u64); + impl quinn::congestion::Controller for FixedWindow { + fn on_congestion_event(&mut self, _: Instant, _: Instant, _: bool, _: u64) {} + fn on_mtu_update(&mut self, _: u16) {} + fn window(&self) -> u64 { + self.0 + } + fn clone_box(&self) -> Box { + Box::new(self.clone()) + } + fn initial_window(&self) -> u64 { + self.0 + } + fn into_any(self: Box) -> Box { + self + } + } + struct Factory; + impl quinn::congestion::ControllerFactory for Factory { + fn build(self: Arc, _: Instant, _: u16) -> Box { + Box::new(FixedWindow(256 * 1024)) + } + } + + let mut t = TransportConfig::default(); + t.max_idle_timeout(Some( + IdleTimeout::try_from(Duration::from_secs(10)).expect("10 s"), + )); + t.keep_alive_interval(Some(Duration::from_secs(1))); + let mut ack = AckFrequencyConfig::default(); + ack.ack_eliciting_threshold(VarInt::from_u32(0)) + .max_ack_delay(Some(Duration::from_millis(1))); + t.ack_frequency_config(Some(ack)); + t.initial_rtt(Duration::from_millis(10)); + t.datagram_receive_buffer_size(Some(64 * 1024)); + t.congestion_controller_factory(Arc::new(Factory)); + Arc::new(t) +} + +fn io_err(e: impl std::fmt::Display) -> std::io::Error { + std::io::Error::other(e.to_string()) +} + +fn client_config(tls: rustls::ClientConfig) -> std::io::Result { + let crypto = quinn::crypto::rustls::QuicClientConfig::try_from(tls).map_err(io_err)?; + let mut config = quinn::ClientConfig::new(Arc::new(crypto)); + config.transport_config(transport()); + Ok(config) +} + +impl QuicLink { + pub fn new( + port: u16, + handler: Handler, + netinfo: NetInfo, + code_file: Option, + ) -> std::io::Result> { + use socket2::{Domain, Protocol, Socket, Type}; + // Dual-stack, like the UDP socket + let socket = Socket::new(Domain::IPV6, Type::DGRAM, Some(Protocol::UDP))?; + socket.set_only_v6(false)?; + socket.bind(&SocketAddr::from((Ipv6Addr::UNSPECIFIED, port)).into())?; + let endpoint = quinn::Endpoint::new( + quinn::EndpointConfig::default(), + None, + socket.into(), + Arc::new(quinn::TokioRuntime), + )?; + tracing::info!("ZLOE/QUIC endpoint on UDP {port}"); + Ok(Arc::new(Self { + endpoint, + port, + state: Mutex::new(State::default()), + handler, + netinfo, + code_file, + pairing_open: Mutex::new(None), + })) + } + + /// The network key is known (configured, or after enrollment): derive the network + /// CA, issue our certificate, start accepting peers. Existing connections belong + /// to the previous key and are closed. + pub fn set_network( + self: &Arc, + nwk_key: &[u8; 16], + epan: Eui64, + me: Eui64, + role: Role, + ) -> std::io::Result<()> { + let secrets = NetworkSecrets::derive(nwk_key, epan).map_err(io_err)?; + let identity = secrets.issue(me, role).map_err(io_err)?; + let client = client_config(secrets.client_config(&identity).map_err(io_err)?)?; + let server_tls = secrets.server_config(&identity).map_err(io_err)?; + let crypto = + quinn::crypto::rustls::QuicServerConfig::try_from(server_tls).map_err(io_err)?; + let mut server = quinn::ServerConfig::with_crypto(Arc::new(crypto)); + server.transport_config(transport()); + self.endpoint.set_server_config(Some(server)); + + let start_accepting = { + let mut state = self.state.lock(); + for (_, conn) in state.conns.drain() { + conn.close(VarInt::from_u32(1), b"network key changed"); + } + state.client = Some(client); + state.me = Some(me); + !std::mem::replace(&mut state.accepting, true) + }; + tracing::info!( + "ZLOE/QUIC: identity {} under network CA for {epan:?}", + security::node_name(me) + ); + if role == Role::Coordinator { + self.new_code(); + } + if start_accepting { + let link = Arc::clone(self); + tokio::spawn(async move { link.accept_loop().await }); + } + Ok(()) + } + + fn new_code(&self) { + let code = security::new_enrollment_code(); + tracing::info!("ZLOE/QUIC: enrollment code {code} (one use, 10 minutes)"); + if let Some(file) = &self.code_file { + let _ = std::fs::write(file, format!("{code}\n")); + } + self.state.lock().code = Some((code, Instant::now())); + } + + pub fn set_pairing_open(&self, pairing_open: PairingOpen) { + *self.pairing_open.lock() = Some(pairing_open); + } + + pub fn is_connected(&self, peer: Eui64) -> bool { + self.state.lock().conns.contains_key(&peer) + } + + pub fn connected(&self) -> Vec { + self.state.lock().conns.keys().copied().collect() + } + + /// Send one ZLOE datagram to a connected peer. + pub fn send(&self, peer: Eui64, datagram: &[u8]) -> bool { + let conn = self.state.lock().conns.get(&peer).cloned(); + conn.is_some_and(|c| c.send_datagram(datagram.to_vec().into()).is_ok()) + } + + /// Dial a peer if we are the lower EUI64 and have no connection or dial underway. + pub fn ensure_connected(self: &Arc, peer: Eui64, addr: SocketAddr) { + let client = { + let mut state = self.state.lock(); + let Some(me) = state.me else { return }; + if me.to_be_bytes() >= peer.to_be_bytes() + || state.conns.contains_key(&peer) + || state.dialing.contains(&peer) + { + return; + } + let Some(client) = state.client.clone() else { + return; + }; + state.dialing.insert(peer); + client + }; + let link = Arc::clone(self); + tokio::spawn(async move { + let result = link.dial(peer, addr, client).await; + link.state.lock().dialing.remove(&peer); + if let Err(e) = result { + tracing::debug!("ZLOE/QUIC: dial {peer:?} at {addr} failed: {e}"); + } + }); + } + + async fn dial( + self: &Arc, + peer: Eui64, + addr: SocketAddr, + client: quinn::ClientConfig, + ) -> std::io::Result<()> { + let conn = self + .endpoint + .connect_with(client, addr, &security::node_name(peer)) + .map_err(io_err)? + .await + .map_err(io_err)?; + // Tell the acceptor who we claim to be; it checks our certificate names it + let me = self.state.lock().me.ok_or_else(|| io_err("no identity"))?; + let mut hello = conn.open_uni().await.map_err(io_err)?; + hello.write_all(&me.0).await.map_err(io_err)?; + hello.finish().map_err(io_err)?; + tracing::info!("ZLOE/QUIC: connected to {peer:?} at {addr}"); + self.adopt(peer, conn); + Ok(()) + } + + async fn accept_loop(self: Arc) { + while let Some(incoming) = self.endpoint.accept().await { + let link = Arc::clone(&self); + tokio::spawn(async move { + if let Err(e) = link.accept(incoming).await { + tracing::warn!("ZLOE/QUIC: rejected connection: {e}"); + } + }); + } + } + + async fn accept(self: &Arc, incoming: quinn::Incoming) -> std::io::Result<()> { + let conn = incoming.await.map_err(io_err)?; + let alpn = conn + .handshake_data() + .and_then(|h| h.downcast::().ok()) + .and_then(|h| h.protocol); + match alpn.as_deref() { + Some(ALPN_ZLOE) => { + let leaf = conn + .peer_identity() + .and_then(|id| id.downcast::>>().ok()) + .and_then(|chain| chain.first().cloned()); + let Some(leaf) = leaf else { + conn.close(VarInt::from_u32(2), b"certificate required"); + return Err(io_err("peer sent no certificate")); + }; + let mut hello = conn.accept_uni().await.map_err(io_err)?; + let claimed = hello.read_to_end(8).await.map_err(io_err)?; + let claimed = Eui64(claimed.try_into().map_err(|_| io_err("bad hello"))?); + if !cert_names(&leaf, claimed) { + conn.close(VarInt::from_u32(3), b"certificate does not name you"); + return Err(io_err(format!("certificate does not name {claimed:?}"))); + } + tracing::info!( + "ZLOE/QUIC: accepted {claimed:?} from {}", + conn.remote_address() + ); + self.adopt(claimed, conn); + Ok(()) + } + Some(ALPN_ENROLL) => self.serve_enrollment(conn).await, + _ => { + conn.close(VarInt::from_u32(4), b"unknown protocol"); + Err(io_err("unknown ALPN")) + } + } + } + + /// Keep the connection and read its datagrams until it closes. + fn adopt(self: &Arc, peer: Eui64, conn: quinn::Connection) { + if let Some(old) = self.state.lock().conns.insert(peer, conn.clone()) { + old.close(VarInt::from_u32(5), b"replaced"); + } + let link = Arc::clone(self); + tokio::spawn(async move { + loop { + match conn.read_datagram().await { + Ok(datagram) => { + if let Some(reply) = (link.handler)(&datagram, conn.remote_address()) { + let _ = conn.send_datagram(reply.into()); + } + } + Err(e) => { + tracing::info!("ZLOE/QUIC: link to {peer:?} closed: {e}"); + let mut state = link.state.lock(); + if state + .conns + .get(&peer) + .is_some_and(|c| c.stable_id() == conn.stable_id()) + { + state.conns.remove(&peer); + } + return; + } + } + } + }); + } + + // -- enrollment ------------------------------------------------------------------ + + async fn serve_enrollment(&self, conn: quinn::Connection) -> std::io::Result<()> { + let peer = conn.remote_address(); + let (mut send, mut recv) = conn.accept_bi().await.map_err(io_err)?; + let proof = recv.read_to_end(64).await.map_err(io_err)?; + let exporter = exporter(&conn).ok_or_else(|| io_err("no exporter"))?; + + let code = { + let state = self.state.lock(); + state + .code + .as_ref() + .filter(|(_, issued)| issued.elapsed() < ENROLL_CODE_LIFETIME) + .map(|(code, _)| code.clone()) + }; + // The current code, or no code at all while the network is in pairing mode + let pairing = self.pairing_open.lock().clone().is_some_and(|open| open()); + let code = if let Some(code) = code.filter(|c| proof_matches(c, &exporter, b'C', &proof)) { + code + } else if pairing && proof_matches(AUTO_CODE, &exporter, b'C', &proof) { + tracing::info!("ZLOE/QUIC: auto-enrolling {peer} (pairing mode is open)"); + AUTO_CODE.to_string() + } else { + tracing::warn!( + "ZLOE/QUIC: enrollment from {peer} rejected (wrong or expired code, or \ + pairing mode closed)" + ); + conn.close(VarInt::from_u32(6), b"enrollment rejected"); + return Ok(()); + }; + let Some(info) = (self.netinfo)() else { + conn.close(VarInt::from_u32(7), b"no network"); + return Ok(()); + }; + // One use: the next satellite needs the next code + if !code.is_empty() { + self.new_code(); + } + + let mut reply = enrollment_proof(&code, &exporter, b'S').to_vec(); + reply.extend_from_slice(&info); + send.write_all(&reply).await.map_err(io_err)?; + send.finish().map_err(io_err)?; + let _ = send.stopped().await; + tracing::info!("ZLOE/QUIC: enrolled {peer}"); + Ok(()) + } + + /// Satellite: prove the code to the coordinator at `addr`, check its proof, and + /// return the network settings it sends. + pub async fn enroll(&self, addr: SocketAddr, code: &str) -> std::io::Result> { + let client = client_config(security::enroll_client_config().map_err(io_err)?)?; + let conn = self + .endpoint + .connect_with(client, addr, "zloe-enroll") + .map_err(io_err)? + .await + .map_err(io_err)?; + let exporter = exporter(&conn).ok_or_else(|| io_err("no exporter"))?; + let (mut send, mut recv) = conn.open_bi().await.map_err(io_err)?; + send.write_all(&enrollment_proof(code, &exporter, b'C')) + .await + .map_err(io_err)?; + send.finish().map_err(io_err)?; + let reply = recv.read_to_end(4096).await.map_err(io_err)?; + conn.close(VarInt::from_u32(0), b"done"); + if reply.len() < 32 || !proof_matches(code, &exporter, b'S', &reply[..32]) { + return Err(io_err("coordinator did not prove the code")); + } + Ok(reply[32..].to_vec()) + } +} diff --git a/crates/ziggurat-phy-trel/src/security.rs b/crates/ziggurat-phy-trel/src/security.rs new file mode 100644 index 0000000..6114fec --- /dev/null +++ b/crates/ziggurat-phy-trel/src/security.rs @@ -0,0 +1,365 @@ +//! SPIKE: ZLOE security material (see BROADCAST.md, "Secure ZLOE over QUIC"). +//! +//! Every secret is derived from the Zigbee network key, salted with the extended PAN +//! ID, so nodes stay stateless: the network CA is the same on every member, and +//! rotating the network key revokes everything. + +use std::sync::Arc; + +use rcgen::{ + BasicConstraints, CertificateParams, DistinguishedName, DnType, IsCa, KeyPair, KeyUsagePurpose, + PKCS_ED25519, SanType, SerialNumber, +}; +use ring::{hkdf, hmac, rand::SecureRandom}; +use rustls::pki_types::{CertificateDer, PrivateKeyDer, PrivatePkcs8KeyDer, ServerName}; +use ziggurat_ieee_802154::types::Eui64; + +pub const ALPN_ZLOE: &[u8] = b"zloe/1"; +pub const ALPN_ENROLL: &[u8] = b"zloe-enroll/1"; +const EXPORTER_LABEL: &[u8] = b"EXPORTER-zloe-enroll"; + +/// The node's role, written into its certificate. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Role { + Coordinator, + Satellite, +} + +/// Everything a network member derives from the network key. +pub struct NetworkSecrets { + pub epan: Eui64, + ca_cert: rcgen::Certificate, + ca_key: KeyPair, + /// Group key for sealed multicast (not used by the PoC yet) + pub mcast_key: [u8; 32], +} + +/// This node's identity on the backbone: an ephemeral key with a certificate issued +/// by the network CA. +pub struct NodeIdentity { + pub cert: CertificateDer<'static>, + key: PrivatePkcs8KeyDer<'static>, +} + +struct Len(usize); +impl hkdf::KeyType for Len { + fn len(&self) -> usize { + self.0 + } +} + +fn hkdf32(nwk_key: &[u8; 16], epan: Eui64, info: &[u8]) -> [u8; 32] { + let prk = hkdf::Salt::new(hkdf::HKDF_SHA256, &epan.0).extract(nwk_key); + let mut out = [0u8; 32]; + prk.expand(&[info], Len(32)) + .and_then(|okm| okm.fill(&mut out)) + .expect("32 bytes is a valid HKDF-SHA256 output length"); + out +} + +/// PKCS#8 v1 wrapper for a raw Ed25519 seed (RFC 8410). +fn ed25519_pkcs8(seed: &[u8; 32]) -> Vec { + let mut der = vec![ + 0x30, 0x2e, 0x02, 0x01, 0x00, 0x30, 0x05, 0x06, 0x03, 0x2b, 0x65, 0x70, 0x04, 0x22, 0x04, + 0x20, + ]; + der.extend_from_slice(seed); + der +} + +/// The DNS name a node's certificate is valid for: `x.zloe`. +pub fn node_name(eui64: Eui64) -> String { + let hex: String = eui64 + .to_be_bytes() + .iter() + .map(|b| format!("{b:02x}")) + .collect(); + format!("x{hex}.zloe") +} + +fn fixed_validity(params: &mut CertificateParams) { + // MCU satellites have no trusted clock; validity is not what revocation relies on + params.not_before = rcgen::date_time_ymd(2000, 1, 1); + params.not_after = rcgen::date_time_ymd(2100, 1, 1); +} + +impl NetworkSecrets { + pub fn derive(nwk_key: &[u8; 16], epan: Eui64) -> Result { + let seed = hkdf32(nwk_key, epan, b"zloe ca v1"); + let der = ed25519_pkcs8(&seed); + let ca_key = KeyPair::from_pkcs8_der_and_sign_algo( + &PrivatePkcs8KeyDer::from(der.as_slice()), + &PKCS_ED25519, + )?; + + // Every field fixed, Ed25519 signatures deterministic: the same CA certificate on + // every member + let mut params = CertificateParams::new(Vec::::new())?; + let mut dn = DistinguishedName::new(); + let epan_hex: String = epan + .to_be_bytes() + .iter() + .map(|b| format!("{b:02x}")) + .collect(); + dn.push(DnType::CommonName, format!("ZLOE network {epan_hex}")); + params.distinguished_name = dn; + params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained); + params.key_usages = vec![ + KeyUsagePurpose::KeyCertSign, + KeyUsagePurpose::DigitalSignature, + ]; + params.serial_number = Some(SerialNumber::from(1u64)); + fixed_validity(&mut params); + let ca_cert = params.self_signed(&ca_key)?; + + Ok(Self { + epan, + ca_cert, + ca_key, + mcast_key: hkdf32(nwk_key, epan, b"zloe multicast v1"), + }) + } + + pub fn ca_der(&self) -> CertificateDer<'static> { + self.ca_cert.der().clone() + } + + /// Issue a certificate for this node with a fresh key. + pub fn issue(&self, eui64: Eui64, role: Role) -> Result { + let key = KeyPair::generate_for(&PKCS_ED25519)?; + let mut params = CertificateParams::new(vec![node_name(eui64)])?; + let mut dn = DistinguishedName::new(); + dn.push( + DnType::CommonName, + format!( + "{} {}", + match role { + Role::Coordinator => "coordinator", + Role::Satellite => "satellite", + }, + node_name(eui64) + ), + ); + params.distinguished_name = dn; + params.subject_alt_names = vec![SanType::DnsName(node_name(eui64).try_into()?)]; + params.key_usages = vec![KeyUsagePurpose::DigitalSignature]; + fixed_validity(&mut params); + let cert = params.signed_by(&key, &self.ca_cert, &self.ca_key)?; + Ok(NodeIdentity { + cert: cert.der().clone(), + key: PrivatePkcs8KeyDer::from(key.serialize_der()), + }) + } + + fn roots(&self) -> rustls::RootCertStore { + let mut roots = rustls::RootCertStore::empty(); + roots + .add(self.ca_der()) + .expect("the network CA is a valid trust anchor"); + roots + } + + /// TLS for connections we dial to a peer: verify it against the network CA under + /// `x.zloe`, present our certificate. + pub fn client_config(&self, me: &NodeIdentity) -> Result { + let mut config = rustls::ClientConfig::builder_with_provider(provider()) + .with_protocol_versions(&[&rustls::version::TLS13])? + .with_root_certificates(self.roots()) + .with_client_auth_cert(vec![me.cert.clone()], me.key())?; + config.alpn_protocols = vec![ALPN_ZLOE.to_vec()]; + Ok(config) + } + + /// TLS for connections we accept: peers must present a certificate from the network + /// CA (ALPN `zloe/1`); enrollment (`zloe-enroll/1`) has none, and is checked by its + /// code proofs instead. + pub fn server_config(&self, me: &NodeIdentity) -> Result { + let verifier = rustls::server::WebPkiClientVerifier::builder_with_provider( + Arc::new(self.roots()), + provider(), + ) + .allow_unauthenticated() + .build() + .map_err(|e| rustls::Error::General(e.to_string()))?; + let mut config = rustls::ServerConfig::builder_with_provider(provider()) + .with_protocol_versions(&[&rustls::version::TLS13])? + .with_client_cert_verifier(verifier) + .with_single_cert(vec![me.cert.clone()], me.key())?; + config.alpn_protocols = vec![ALPN_ZLOE.to_vec(), ALPN_ENROLL.to_vec()]; + Ok(config) + } +} + +impl NodeIdentity { + fn key(&self) -> PrivateKeyDer<'static> { + PrivateKeyDer::Pkcs8(self.key.clone_key()) + } +} + +/// Whether `cert` (already verified to chain to the network CA by TLS) names `eui64`. +pub fn cert_names(cert: &CertificateDer<'_>, eui64: Eui64) -> bool { + let Ok(end_entity) = webpki::EndEntityCert::try_from(cert) else { + return false; + }; + let Ok(name) = ServerName::try_from(node_name(eui64)) else { + return false; + }; + end_entity.verify_is_valid_for_subject_name(&name).is_ok() +} + +pub fn provider() -> Arc { + Arc::new(rustls::crypto::ring::default_provider()) +} + +// -- enrollment --------------------------------------------------------------------- + +/// A fresh one-time enrollment code: 128 random bits, base32 (26 characters). +pub fn new_enrollment_code() -> String { + let mut bytes = [0u8; 16]; + ring::rand::SystemRandom::new() + .fill(&mut bytes) + .expect("system randomness"); + base32(&bytes) +} + +fn base32(bytes: &[u8]) -> String { + const ALPHABET: &[u8; 32] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZ234567"; + let mut out = String::new(); + let (mut buffer, mut bits) = (0u32, 0u32); + for &b in bytes { + buffer = (buffer << 8) | u32::from(b); + bits += 8; + while bits >= 5 { + bits -= 5; + out.push(ALPHABET[((buffer >> bits) & 31) as usize] as char); + } + } + if bits > 0 { + out.push(ALPHABET[((buffer << (5 - bits)) & 31) as usize] as char); + } + out +} + +/// TLS session binding for enrollment proofs (RFC 5705 exporter). +pub fn exporter(conn: &quinn::Connection) -> Option<[u8; 32]> { + let mut out = [0u8; 32]; + conn.export_keying_material(&mut out, EXPORTER_LABEL, b"") + .ok()?; + Some(out) +} + +/// `HMAC-SHA256(code, exporter ‖ side)`: proves knowledge of the code, bound to this +/// TLS session, so a man in the middle (two sessions, no code) can produce neither. +pub fn enrollment_proof(code: &str, exporter: &[u8; 32], side: u8) -> [u8; 32] { + let key = hmac::Key::new(hmac::HMAC_SHA256, code.trim().to_uppercase().as_bytes()); + let mut msg = exporter.to_vec(); + msg.push(side); + let tag = hmac::sign(&key, &msg); + let mut out = [0u8; 32]; + out.copy_from_slice(tag.as_ref()); + out +} + +pub fn proof_matches(code: &str, exporter: &[u8; 32], side: u8, proof: &[u8]) -> bool { + let key = hmac::Key::new(hmac::HMAC_SHA256, code.trim().to_uppercase().as_bytes()); + let mut msg = exporter.to_vec(); + msg.push(side); + hmac::verify(&key, &msg, proof).is_ok() +} + +/// TLS for the enrolling satellite: it has no network CA yet, so it accepts any server +/// certificate (signatures still checked); the code proofs authenticate the server. +pub fn enroll_client_config() -> Result { + let mut config = rustls::ClientConfig::builder_with_provider(provider()) + .with_protocol_versions(&[&rustls::version::TLS13])? + .dangerous() + .with_custom_certificate_verifier(Arc::new(AnyServer(provider()))) + .with_no_client_auth(); + config.alpn_protocols = vec![ALPN_ENROLL.to_vec()]; + Ok(config) +} + +#[derive(Debug)] +struct AnyServer(Arc); + +impl rustls::client::danger::ServerCertVerifier for AnyServer { + fn verify_server_cert( + &self, + _end_entity: &CertificateDer<'_>, + _intermediates: &[CertificateDer<'_>], + _server_name: &ServerName<'_>, + _ocsp_response: &[u8], + _now: rustls::pki_types::UnixTime, + ) -> Result { + Ok(rustls::client::danger::ServerCertVerified::assertion()) + } + + fn verify_tls12_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &rustls::DigitallySignedStruct, + ) -> Result { + rustls::crypto::verify_tls12_signature( + message, + cert, + dss, + &self.0.signature_verification_algorithms, + ) + } + + fn verify_tls13_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &rustls::DigitallySignedStruct, + ) -> Result { + rustls::crypto::verify_tls13_signature( + message, + cert, + dss, + &self.0.signature_verification_algorithms, + ) + } + + fn supported_verify_schemes(&self) -> Vec { + self.0.signature_verification_algorithms.supported_schemes() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn same_network_key_same_ca() { + let epan = Eui64([1, 2, 3, 4, 5, 6, 7, 8]); + let a = NetworkSecrets::derive(&[7; 16], epan).unwrap(); + let b = NetworkSecrets::derive(&[7; 16], epan).unwrap(); + let c = NetworkSecrets::derive(&[8; 16], epan).unwrap(); + assert_eq!(a.ca_der(), b.ca_der()); + assert_eq!(a.mcast_key, b.mcast_key); + assert_ne!(a.ca_der(), c.ca_der()); + } + + #[test] + fn node_cert_names_its_eui64() { + let secrets = NetworkSecrets::derive(&[7; 16], Eui64([9; 8])).unwrap(); + let me = Eui64([0x20, 1, 2, 3, 4, 5, 6, 7]); + let id = secrets.issue(me, Role::Satellite).unwrap(); + assert!(cert_names(&id.cert, me)); + assert!(!cert_names(&id.cert, Eui64([0x21; 8]))); + } + + #[test] + fn enrollment_proofs() { + let code = new_enrollment_code(); + assert_eq!(code.len(), 26); + let exp = [3u8; 32]; + let proof = enrollment_proof(&code, &exp, b'C'); + assert!(proof_matches(&code, &exp, b'C', &proof)); + assert!(!proof_matches(&code, &exp, b'S', &proof)); + assert!(!proof_matches("WRONGCODE", &exp, b'C', &proof)); + assert!(!proof_matches(&code, &[4u8; 32], b'C', &proof)); + } +} diff --git a/crates/ziggurat-phy-trel/tests/loopback.rs b/crates/ziggurat-phy-trel/tests/loopback.rs index 3761417..b624d28 100644 --- a/crates/ziggurat-phy-trel/tests/loopback.rs +++ b/crates/ziggurat-phy-trel/tests/loopback.rs @@ -122,6 +122,7 @@ async fn node( mcast_port: 0, mcast_block_file: None, ipv4_only: false, + quic: None, deny: DenyList::default(), }, ) diff --git a/crates/ziggurat-phy-trel/tests/quic.rs b/crates/ziggurat-phy-trel/tests/quic.rs new file mode 100644 index 0000000..5f44062 --- /dev/null +++ b/crates/ziggurat-phy-trel/tests/quic.rs @@ -0,0 +1,290 @@ +//! SPIKE: secure ZLOE (QUIC) between two nodes over loopback, each with a fake radio. + +use std::sync::Arc; +use std::time::Duration; + +use parking_lot::Mutex; +use tokio::sync::mpsc; +use ziggurat_ieee_802154::types::{Eui64, Nwk, PanId}; +use ziggurat_phy::*; +use ziggurat_phy_trel::{DenyList, TrelConfig, TrelPhy}; + +#[derive(Default)] +struct FakeState { + sent: Mutex>>, + rx: Mutex>>, +} + +struct FakeRadio(Arc); + +struct Rx(mpsc::Receiver); +impl Receiver for Rx { + async fn recv(&mut self) -> Option { + self.0.recv().await + } +} +struct NoExclusive; +impl ExclusiveRadio for NoExclusive { + async fn set_channel(&self, _: u8) -> Result<(), RadioError> { + Ok(()) + } + async fn transmit(&self, _: TxFrame) -> Result { + Ok(TxResult::Acked) + } +} + +impl RadioPhy for FakeRadio { + type Exclusive<'a> = NoExclusive; + type RxStream = Rx; + type ResetStream = Rx; + async fn reset(&self) -> Result<(), RadioError> { + Ok(()) + } + async fn reconfigure(&self, _: &RadioConfig) -> Result<(), RadioError> { + Ok(()) + } + async fn set_frame_pending_table(&self, _: &[Nwk], _: &[Eui64]) -> Result<(), RadioError> { + Ok(()) + } + async fn transmit(&self, frame: TxFrame) -> Result { + self.0.sent.lock().push(frame.psdu); + Ok(TxResult::NoAck) + } + async fn energy_detect(&self, _: u8, _: Duration) -> Result { + Ok(0) + } + async fn lock(&self) -> NoExclusive { + NoExclusive + } + fn subscribe_rx(&self) -> Rx { + let (tx, rx) = mpsc::channel(8); + *self.0.rx.lock() = Some(tx); + Rx(rx) + } + fn subscribe_reset(&self) -> Rx { + Rx(mpsc::channel(1).1) + } +} + +fn config(eui: u8, nwk: u16) -> RadioConfig { + RadioConfig { + channel: 15, + tx_power: 0, + short_address: Nwk(nwk), + extended_address: Eui64([eui; 8]), + pan_id: PanId(0x1234), + promiscuous: false, + rx_on_when_idle: true, + frame_pending_short: vec![], + frame_pending_extended: vec![], + } +} + +/// Data frame, short dst/src, PAN compression, ack request as given. +fn data_frame(dst: u16, src: u16, ack: bool) -> TxFrame { + let fcf: u16 = 0x0001 | (u16::from(ack) << 5) | (1 << 6) | (2 << 10) | (2 << 14); + let mut psdu = fcf.to_le_bytes().to_vec(); + psdu.push(7); + psdu.extend_from_slice(&0x1234u16.to_le_bytes()); + psdu.extend_from_slice(&dst.to_le_bytes()); + psdu.extend_from_slice(&src.to_le_bytes()); + psdu.extend_from_slice(b"hello"); + TxFrame { + psdu, + channel: None, + csma_ca: true, + max_frame_retries: 3, + max_csma_backoffs: 4, + security_processed: true, + } +} + +use ziggurat_phy_trel::QuicConfig; + +const NWK_KEY: [u8; 16] = [0x11; 16]; + +async fn secure_node( + epan: Eui64, + port: u16, + eui: u8, + nwk: u16, + key: Option<[u8; 16]>, +) -> (TrelPhy, Arc) { + let radio = Arc::new(FakeState::default()); + let phy = TrelPhy::new( + FakeRadio(Arc::clone(&radio)), + TrelConfig { + port, + static_peers: vec![], + mdns: true, + interface: None, + prefer_ipv4: false, + mcast_port: 0, + mcast_block_file: None, + ipv4_only: false, + quic: Some(QuicConfig { + port: port + 1, + code_file: None, + }), + deny: DenyList::default(), + }, + ) + .await + .unwrap(); + phy.reconfigure(&config(eui, nwk)).await.unwrap(); + phy.set_extended_pan_id(epan); + if let Some(key) = key { + phy.set_network_key(&key, epan, Eui64([eui; 8]), nwk == 0) + .unwrap(); + } + (phy, radio) +} + +/// Send until acked over QUIC (the link needs mDNS discovery plus a handshake first). +async fn send_until_acked(phy: &TrelPhy, radio: &FakeState, dst: u16) -> bool { + let deadline = tokio::time::Instant::now() + Duration::from_secs(20); + while tokio::time::Instant::now() < deadline { + let before = radio.sent.lock().len(); + let result = phy.transmit(data_frame(dst, 0x0000, true)).await.unwrap(); + // Acked without touching the radio: it went over QUIC + if result == TxResult::Acked && radio.sent.lock().len() == before { + return true; + } + tokio::time::sleep(Duration::from_millis(250)).await; + } + false +} + +#[tokio::test(flavor = "multi_thread")] +async fn secure_peers_exchange_over_quic() { + let epan = Eui64([0x51; 8]); + let (a, radio_a) = secure_node(epan, 47301, 0xA1, 0x0000, Some(NWK_KEY)).await; + let (b, _radio_b) = secure_node(epan, 47303, 0xB1, 0x2222, Some(NWK_KEY)).await; + let mut rx_b = b.subscribe_rx(); + assert!( + send_until_acked(&a, &radio_a, 0x2222).await, + "peers: {:?}", + a.peers() + ); + let got = tokio::time::timeout(Duration::from_secs(1), rx_b.recv()) + .await + .unwrap() + .unwrap(); + assert_eq!(&got.psdu[got.psdu.len() - 5..], b"hello"); +} + +#[tokio::test(flavor = "multi_thread")] +async fn other_network_key_is_refused() { + let epan = Eui64([0x52; 8]); + let (a, radio_a) = secure_node(epan, 47311, 0xA2, 0x0000, Some(NWK_KEY)).await; + let (_b, _) = secure_node(epan, 47313, 0xB2, 0x3333, Some([0x22; 16])).await; + // B is discovered as a secure peer (same epan), so only TLS can keep it out + let found = tokio::time::Instant::now() + Duration::from_secs(10); + while !a.peers().iter().any(|p| p.contains("b2:b2")) { + assert!(tokio::time::Instant::now() < found, "B never discovered"); + tokio::time::sleep(Duration::from_millis(100)).await; + } + // Never acked over QUIC: TLS fails, every send falls back to the radio + let deadline = tokio::time::Instant::now() + Duration::from_secs(8); + while tokio::time::Instant::now() < deadline { + let before = radio_a.sent.lock().len(); + a.transmit(data_frame(0x3333, 0x0000, true)).await.unwrap(); + assert_eq!( + radio_a.sent.lock().len(), + before + 1, + "sent over the backbone" + ); + tokio::time::sleep(Duration::from_millis(250)).await; + } +} + +async fn coordinator_with_settings(epan: Eui64, port: u16, eui: u8) -> TrelPhy { + let (coord, _) = secure_node(epan, port, eui, 0x0000, Some(NWK_KEY)).await; + coord.set_netinfo_provider(Arc::new(|| Some(b"network settings".to_vec()))); + coord +} + +#[tokio::test(flavor = "multi_thread")] +async fn enrollment_with_code() { + let epan = Eui64([0x53; 8]); + let dir = std::env::temp_dir().join(format!("zloe-enroll-{}", std::process::id())); + let code_file = dir.with_extension("code"); + let coord = { + let radio = Arc::new(FakeState::default()); + let phy = TrelPhy::new( + FakeRadio(radio), + TrelConfig { + port: 47321, + static_peers: vec![], + mdns: true, + interface: None, + prefer_ipv4: false, + mcast_port: 0, + mcast_block_file: None, + ipv4_only: false, + quic: Some(QuicConfig { + port: 47322, + code_file: Some(code_file.clone()), + }), + deny: DenyList::default(), + }, + ) + .await + .unwrap(); + phy.reconfigure(&config(0xA3, 0x0000)).await.unwrap(); + phy.set_extended_pan_id(epan); + phy.set_network_key(&NWK_KEY, epan, Eui64([0xA3; 8]), true) + .unwrap(); + phy.set_netinfo_provider(Arc::new(|| Some(b"network settings".to_vec()))); + phy + }; + let code = std::fs::read_to_string(&code_file).unwrap(); + // Satellite: no network key yet + let (sat, _) = secure_node(epan, 47323, 0xB3, 0x4444, None).await; + + let wrong = sat + .enroll( + Some(epan), + "AAAAAAAAAAAAAAAAAAAAAAAAAA", + Duration::from_secs(10), + ) + .await; + assert!(wrong.is_err(), "wrong code accepted"); + let settings = sat + .enroll(Some(epan), code.trim(), Duration::from_secs(10)) + .await + .unwrap(); + assert_eq!(settings, b"network settings"); + // One use + assert!( + sat.enroll(Some(epan), code.trim(), Duration::from_secs(5)) + .await + .is_err() + ); + drop(coord); +} + +#[tokio::test(flavor = "multi_thread")] +async fn auto_enrollment_only_while_pairing() { + let epan = Eui64([0x54; 8]); + let coord = coordinator_with_settings(epan, 47331, 0xA4).await; + let pairing = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let flag = Arc::clone(&pairing); + coord.set_pairing_open(Arc::new(move || { + flag.load(std::sync::atomic::Ordering::Relaxed) + })); + let (sat, _) = secure_node(epan, 47333, 0xB4, 0x5555, None).await; + + assert!( + sat.enroll(Some(epan), "", Duration::from_secs(10)) + .await + .is_err(), + "closed" + ); + pairing.store(true, std::sync::atomic::Ordering::Relaxed); + let settings = sat + .enroll(Some(epan), "", Duration::from_secs(10)) + .await + .unwrap(); + assert_eq!(settings, b"network settings"); +} diff --git a/crates/ziggurat-server/src/main.rs b/crates/ziggurat-server/src/main.rs index 006b159..b917061 100644 --- a/crates/ziggurat-server/src/main.rs +++ b/crates/ziggurat-server/src/main.rs @@ -689,6 +689,15 @@ impl ZigguratServer { let aps_frame_counter = payload.state.aps_frame_counter; // SPIKE: ZLOE peers are scoped by the extended PAN ID (TREL's `xp`) phy.set_extended_pan_id(payload.state.extended_pan_id); + // SPIKE: secure ZLOE derives its network CA from the network key + if let Err(e) = phy.set_network_key( + &payload.state.network_key.0, + payload.state.extended_pan_id, + payload.state.ieee_address, + payload.role == proto::NodeRole::Coordinator, + ) { + tracing::warn!("ZLOE/QUIC: cannot set up security: {e}"); + } let stack = ZigbeeStack::new( phy, proto::network_config(&payload), @@ -725,6 +734,14 @@ impl ZigguratServer { arc_self.run().await; }); + // SPIKE: secure ZLOE accepts enrollment without a code while joins are open + let pairing_stack = Arc::downgrade(&stack); + stack.radio.set_pairing_open(Arc::new(move || { + pairing_stack + .upgrade() + .is_some_and(|s| s.trust_center_permitting_joins()) + })); + // SPIKE: on the trusted ZLOE backbone a coordinator hands its settings to any // satellite that asks let weak = Arc::downgrade(&stack); @@ -1058,6 +1075,25 @@ struct Args { #[arg(long)] zloe_test_mcast_block_file: Option, + /// SPIKE: ZLOE security: `none` (trusted backbone) or `quic` (authenticated + /// QUIC peer links, keys derived from the network key) + #[arg(long, default_value = "none")] + zloe_security: String, + + /// SPIKE: UDP port of the ZLOE QUIC endpoint (default: --zloe-port + 1) + #[arg(long, default_value_t = 0)] + zloe_quic_port: u16, + + /// SPIKE: with --zloe-satellite and --zloe-security quic, enroll with this code + /// from the coordinator, or `auto` while the network is in pairing mode + #[arg(long)] + zloe_enroll: Option, + + /// SPIKE: coordinator in --zloe-security quic: write the current enrollment code + /// to this file + #[arg(long)] + zloe_enroll_code_file: Option, + /// SPIKE test hook: behave like a host without IPv6 #[arg(long)] zloe_test_ipv4_only: bool, @@ -1158,6 +1194,22 @@ fn main() -> Result<(), Box> { mcast_port: args.zloe_mcast_port, mcast_block_file: args.zloe_test_mcast_block_file, ipv4_only: args.zloe_test_ipv4_only, + quic: match args.zloe_security.as_str() { + "quic" => Some(ziggurat_phy_trel::QuicConfig { + port: if args.zloe_quic_port != 0 { + args.zloe_quic_port + } else { + args.zloe_port + 1 + }, + code_file: args.zloe_enroll_code_file.clone(), + }), + "none" => None, + other => { + return Err( + format!("--zloe-security must be none or quic, not {other:?}").into(), + ); + } + }, deny, }, )); @@ -1218,9 +1270,14 @@ fn main() -> Result<(), Box> { .as_deref() .map(Eui64::try_from_hex) .transpose()?; + let enroll = args.zloe_enroll.clone(); let server = server.clone(); tokio::spawn(async move { - if let Err(e) = satellite::bootstrap_zloe(&server, epan, nwk).await { + let result = match enroll { + Some(code) => satellite::bootstrap_enroll(&server, epan, nwk, &code).await, + None => satellite::bootstrap_zloe(&server, epan, nwk).await, + }; + if let Err(e) = result { tracing::error!("Satellite bootstrap failed: {e}"); std::process::exit(1); } diff --git a/crates/ziggurat-server/src/satellite.rs b/crates/ziggurat-server/src/satellite.rs index fa5d534..3a1fd8a 100644 --- a/crates/ziggurat-server/src/satellite.rs +++ b/crates/ziggurat-server/src/satellite.rs @@ -36,6 +36,36 @@ pub async fn bootstrap_zloe( start_router(server, &info, nwk).await } +/// Secure mode: enroll with the coordinator over QUIC (with a code, or `auto` while the +/// network is in pairing mode) and start with the settings it sends. +pub async fn bootstrap_enroll( + server: &std::sync::Arc, + epan: Option, + nwk: Option, + code: &str, +) -> Result<(), Box> { + let code = if code == "auto" { + ziggurat_phy_trel::quic::AUTO_CODE + } else { + code + }; + tracing::info!( + "Satellite: enrolling over ZLOE/QUIC ({})", + if code.is_empty() { + "pairing mode" + } else { + "code" + } + ); + let phy = server.phy().await?; + let bytes = phy + .enroll(epan, code, std::time::Duration::from_secs(60)) + .await?; + let info: proto::NetworkInfoPayload = + proto::parse(&bytes).ok_or("malformed network settings from coordinator")?; + start_router(server, &info, nwk).await +} + async fn start_router( server: &std::sync::Arc, info: &proto::NetworkInfoPayload, From 9b6af57e2ff680a5cdab2ac945db1a7ccece8211 Mon Sep 17 00:00:00 2001 From: Igor Yamolov Date: Tue, 10 Nov 2026 18:24:25 +0100 Subject: [PATCH 19/22] SPIKE: soak report (ZHA, 3 backbone nodes, 52 min) Co-Authored-By: Claude Opus 5.5 --- crates/ziggurat-phy-trel/SPIKE.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/crates/ziggurat-phy-trel/SPIKE.md b/crates/ziggurat-phy-trel/SPIKE.md index 5ace0a8..757692f 100644 --- a/crates/ziggurat-phy-trel/SPIKE.md +++ b/crates/ziggurat-phy-trel/SPIKE.md @@ -127,3 +127,25 @@ devices. - Bench: bootstrap and backbone over `fe80::…%enp4s0f0`; deny run 4/4 via the satellite, 41–64 ms. Known gap: a peer's address locks to its datagrams' source, so `--zloe-ipv4` on one side is overridden when the other side sends over IPv6. +## Soak (ZHA, 3 backbone nodes, multicast) + +**Setup:** +- Home Assistant 2026.11.1 with ZHA, coordinator plus two satellites on a wired backbone (IPv6 link-local), multicast on. +- One On/Off every 30 s to the ESP32-C6 ZBOSS router `0x93E4`, sent through the coordinator's API alongside ZHA. + +**Window:** 17:31–18:23, 52.5 minutes. + +| Metric | Result | +|---|---| +| On/Off sent | 106 | +| Acknowledged (APS ack) | **106 (100 %)** | +| Latency to APS ack | min 40 ms, median 41 ms, p95 50 ms, p99 66 ms, max 66 ms | +| Backbone broadcasts (3 nodes, 45 min) | 542, all sent as one multicast datagram, **0** unicast copies | +| Multicast probes (3 nodes, 45 min) | 1626, preferred group only (one every 5 s per node) | +| Backbone links | coordinator and satellites at cost 1/1 in every 5-minute snapshot | +| Radio links to the ZBOSS router | stable at in-cost 7, out-cost 4–5 (ESP RCP LQI ~10) | + +**Observations:** +- No failed send, no neighbour lost, and no fallback to unicast copies during the whole run. +- The router is a direct radio neighbour of the coordinator, so the On/Off traffic stays on the radio. The backbone carries the mesh's own traffic: Link Status, many-to-one route requests, ZHA's topology scans. +- Probe traffic is now about 3× the broadcast traffic on this quiet network (1626 against 542). Piggybacking the ack request on real broadcasts would remove most of it. From f6faddc4e1d16ba2abe0f4e3b92e7aa488b18a35 Mon Sep 17 00:00:00 2001 From: Igor Yamolov Date: Tue, 10 Nov 2026 18:32:14 +0100 Subject: [PATCH 20/22] SPIKE: keep the link-local scope when dialing QUIC peers SocketAddr::new(ip, port) drops an IPv6 scope id, so enrolling with or dialing a link-local peer on its QUIC port timed out on the bench. Co-Authored-By: Claude Opus 5.5 --- crates/ziggurat-phy-trel/src/lib.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/crates/ziggurat-phy-trel/src/lib.rs b/crates/ziggurat-phy-trel/src/lib.rs index 88d2f57..4a865c8 100644 --- a/crates/ziggurat-phy-trel/src/lib.rs +++ b/crates/ziggurat-phy-trel/src/lib.rs @@ -532,7 +532,7 @@ impl TrelPhy

{ shared .coordinator_quic .get(eui64) - .map(|qp| (*eui64, SocketAddr::new(addr.ip(), *qp))) + .map(|qp| (*eui64, with_port(*addr, *qp))) }) }; if let Some((eui64, addr)) = target { @@ -1230,7 +1230,7 @@ impl Backbone { .iter() .filter_map(|(eui64, p)| { p.quic_port - .map(|qp| (*eui64, SocketAddr::new(p.addr.ip(), qp))) + .map(|qp| (*eui64, with_port(p.addr, qp))) }) .collect(); for (peer, addr) in targets { @@ -1862,6 +1862,13 @@ fn multicast_group_v4(epan: Eui64) -> Ipv4Addr { Ipv4Addr::new(239, 255, e[6], e[7]) } +/// The same address on another port, keeping an IPv6 link-local scope +/// (`SocketAddr::new(ip, port)` would drop it). +fn with_port(mut addr: SocketAddr, port: u16) -> SocketAddr { + addr.set_port(port); + addr +} + /// IPv4 targets on a dual-stack IPv6 socket must be v4-mapped. fn to_family(addr: SocketAddr, dual_stack: bool) -> SocketAddr { match addr { From 9235e04d16bff59c555a03d09ec2344e1629250b Mon Sep 17 00:00:00 2001 From: Igor Yamolov Date: Tue, 10 Nov 2026 19:54:37 +0100 Subject: [PATCH 21/22] SPIKE: design and gap analysis for ZLOE satellites on the MCU Four backbone transports for a satellite running Ziggurat on the radio chip: UART relay through the host, Wi-Fi coexistence, SPI Ethernet and esp-hosted. Shared no_std ZloeCore extracted from TrelPhy; the three lwIP transports share one C glue. Design only. Co-Authored-By: Claude Opus 5.5 --- crates/ziggurat-phy-trel/MCU.md | 155 ++++++++++++++++++++++++++++++++ 1 file changed, 155 insertions(+) create mode 100644 crates/ziggurat-phy-trel/MCU.md diff --git a/crates/ziggurat-phy-trel/MCU.md b/crates/ziggurat-phy-trel/MCU.md new file mode 100644 index 0000000..a5955aa --- /dev/null +++ b/crates/ziggurat-phy-trel/MCU.md @@ -0,0 +1,155 @@ +# ZLOE on the MCU: satellites running Ziggurat on the radio chip + +Status: design (2026-11-10). Nothing built yet. Part of the zigpy/ziggurat#84 spike. + +Styleguide: the repository's (`.pre-commit-config.yaml`: `cargo fmt`, clippy +`-D warnings -Dclippy::all -Dclippy::nursery`, `clang-format` for the C/C++ glue). +Rust for the MCU is `no_std` + `alloc` on the embassy runtime, like +`ziggurat-ot`; C glue follows `firmwares/glue`. Tests stay at spike level: unit tests +for the `no_std` core on the host, bench runs for the firmware. + +## Goal + +Today a satellite is a host process (`ziggurat --zloe-satellite`) driving an RCP over +USB: the radio sits wherever the host is. Ziggurat already runs on the chip +(`firmwares/esp32-c6`: Ziggurat as a `no_std` staticlib inside OpenThread RCP, its +only outside link the Spinel vendor tunnel over UART). Goal: a satellite that is the +chip, reaching the ZLOE backbone by one of four transports, and a comparison of them. + +## Architecture + +``` + ziggurat-driver (router role, unchanged) + │ RadioPhy + ┌───────────┴───────────┐ + │ ZloeCore (no_std) │ peer table, link selection, ack/retry, + │ wraps OtLinkPhy │ dedup, LQI from backbone health + └───────────┬───────────┘ (TrelPhy's logic, without tokio/std) + │ BackboneTransport trait + ┌───────────────┴────────────────┐ + UART relay (Spinel tunnel) lwIP on the chip (C glue: UDP + ESP mDNS) + host: thin relay, puts netif = Wi-Fi (coex) | SPI Ethernet (W5500) + datagrams on ZLOE | esp-hosted (second ESP32 over SPI) +``` + +- **ZloeCore** is the part of `TrelPhy` that is not I/O: identity, peer table, + "backbone first, radio on failure", per-frame ack with retries, duplicate drop, + backbone health → LQI. It moves into a `no_std` crate, and `TrelPhy` on the host is + rebuilt on top of it, so both sides keep one implementation. +- **BackboneTransport** is what the core needs from below: send a datagram to a peer + address, deliver received datagrams, report peers found by discovery, report link + up/down. +- **Wire format** stays ZLOE v2, unauthenticated (`--zloe-security none`). QUIC on the + MCU (ngtcp2, ~37 KB heap and ~641 ms per handshake on an ESP32) is a later step. +- **Bootstrap:** the satellite fetches the network settings with `NETINFO_REQ` over + the backbone (as host satellites do), parses `NetworkInfoPayload` (the protocol crate + is `no_std`) and configures itself in router role. NWK address from the EUI64, + frame counters seeded from a clock (SNTP on lwIP transports; the host's time via the + relay). + +## The four transports + +| | UART relay | Wi-Fi (coexistence) | SPI Ethernet | esp-hosted | +|---|---|---|---|---| +| Hardware | none extra | none extra | W5500 (or DM9051) module on SPI | second ESP32 (Wi-Fi) on SPI | +| Network stack | host's | lwIP on the C6 | lwIP on the C6 | lwIP on the C6, Wi-Fi on the slave | +| Radio sharing | none | 802.15.4 and Wi-Fi time-slice one radio | none | none (separate chip) | +| Multicast | host's | unreliable on Wi-Fi: announce none | yes | Wi-Fi: announce none | +| Standalone | no (needs a host) | yes | yes | yes | +| What it tests | on-chip router role | coexistence cost | wired standalone | Wi-Fi without coexistence | + +## Phases + +1. **ZloeCore extraction** (host-tested): move the logic out of `TrelPhy` into a + `no_std` crate with the transport behind a trait; `TrelPhy` uses it, the existing + loopback and QUIC tests stay green. +2. **UART relay:** a Spinel vendor command pair carries ZLOE datagrams between the + chip and the host; the host relay is `ziggurat-phy-trel` in "relay" mode (mDNS and + UDP on behalf of the chip, no Zigbee stack). Firmware: ZloeCore around `OtLinkPhy` + in `ziggurat-ot`. +3. **lwIP glue:** C glue for UDP sockets, ESP mDNS (`_zloe._udp` announce and browse), + SNTP; Rust FFI behind `BackboneTransport`. Brought up on the transport with the + least risk first: SPI Ethernet if a module is available, otherwise Wi-Fi. +4. **Wi-Fi coexistence:** `CONFIG_ESP_COEX_SW_COEXIST_ENABLE`, coexistence priority + to 802.15.4, Wi-Fi power save off. Credentials in a gitignored local header the + user writes. +5. **esp-hosted:** esp-hosted-mcu host component on the C6 over SPI, slave firmware on + the second ESP32. +6. **Bench comparison:** the deny experiment (On/Off to a device only the MCU + satellite hears) and a soak per transport; record delivery, latency, Zigbee frame + loss on the MCU satellite's radio, Wi-Fi throughput where relevant. + +## Gap analysis + +| ID | Gap | Severity | Resolution | +|---|---|---|---| +| G1 | RAM: OT RCP + Ziggurat + lwIP (+ Wi-Fi driver ~100 KB) in the C6's 512 KB SRAM. | must | Measure free heap per phase; Wi-Fi and esp-hosted builds drop OT features the RCP does not need; neighbour/route tables capped on the MCU. | +| G2 | Coexistence: 802.15.4 loses airtime whenever Wi-Fi holds the radio; Zigbee frames from end devices and routers are missed. | must (measure) | Coexistence priority to 802.15.4, power save off, measure frame loss against the Ethernet build. The result decides whether Wi-Fi coexistence is viable at all. | +| G3 | ESP-IDF Wi-Fi defaults to modem sleep, which delays downlink by up to a DTIM interval (spinel2quic measurement). | must | `WIFI_PS_NONE`. | +| G4 | Timing: Zigbee MAC ACKs come from the radio; ZLOE acks (50 ms × 3) run on the MCU executor next to the stack. | minor | Same constants as the host; measure retry rates. | +| G5 | The UART relay adds Spinel framing and host scheduling to every backbone frame. | minor | Latency comparison in phase 6; acceptable for a test transport. | +| G6 | Clock for frame-counter seeding: the MCU has no wall clock at boot. | must | SNTP on lwIP transports before configuring; the relay passes the host time; otherwise the satellite refuses to start rather than reuse counters. | +| G7 | mDNS on the MCU: ESP mDNS component, one interface, IPv4 and IPv6 link-local. | minor | Announce `xa`, `xp`, `nwk`, `role=router`, `sec=none`, no `mc` on Wi-Fi. | +| G8 | Settings fetched with unauthenticated `NETINFO_REQ` (network key in clear on the LAN). | accepted | Same as host satellites in `none` mode; QUIC enrollment on the MCU later. | +| G9 | esp-hosted on the C6 as host: SDIO is slave-only on the C6, so the link to the second ESP32 is SPI (or UART). | must | SPI transport; check esp-hosted-mcu support for an ESP32-C6 host. | +| G10 | SPI Ethernet needs a module (W5500/DM9051) and wiring. | hardware | Confirm availability; pins from the devkit's free GPIOs. | +| G11 | Firmware OTA/updates and partition size (RCP + Ziggurat + lwIP + Wi-Fi). | minor | 4 MB flash is enough for one app partition; no OTA in the spike. | +| G12 | Host relay must not become a second Zigbee stack. | minor | Relay only moves datagrams and runs discovery; all ZLOE decisions stay in ZloeCore on the chip. | + +## Behaviour + +``` +ZloeCore + transmit + Broadcast frame + - transmits it on the radio + - sends one backbone datagram per peer (or per announced multicast group) + Unicast to a backbone peer + - sends it as a backbone datagram requesting an ack + - returns Acked when the ack arrives + No ack after three tries + - transmits it on the radio + - lowers the peer's backbone health + Unicast to an unknown destination + - transmits it on the radio + receive + Datagram from a peer on our extended PAN ID + - delivers the frame to the stack once, LQI from the peer's backbone health + - acks it when asked + Radio frame from a live backbone peer + - drops it + Datagram from another network + - drops it + bootstrap + Coordinator found by discovery + - requests the network settings + - configures the stack in router role + - announces itself with Device_annce + Clock unknown + - does not start +UART relay (host) + forward_up + Datagram from the backbone for the chip + - passes it to the chip over the Spinel tunnel unchanged + forward_down + Datagram from the chip with a destination + - sends it on the backbone from the relay's socket + discovery + Peer found by mDNS + - reports it to the chip +lwIP transport (C glue) + start + Network interface up (Wi-Fi, Ethernet or esp-hosted) + - opens the UDP socket + - announces _zloe._udp with xa, xp, nwk, role, sec + - browses for peers and reports them + Network interface down + - reports link down + - ZloeCore sends everything on the radio +``` + +## Hardware needed + +- New ESP32-C6 (in hand): UART relay and Wi-Fi coexistence. +- SPI Ethernet module (W5500 or DM9051) with jumper wires: SPI Ethernet. +- Second ESP32 with Wi-Fi, wired to the C6 over SPI: esp-hosted. From a7c014ea6400c274253a39b0bd2c4b8d4511d4af Mon Sep 17 00:00:00 2001 From: Igor Yamolov Date: Thu, 12 Nov 2026 13:28:31 +0100 Subject: [PATCH 22/22] SPIKE: drop the #88 next-hop workaround, upstream #89 covers it; cargo fmt Co-Authored-By: Claude Opus 5.5 --- crates/ziggurat-phy-trel/src/lib.rs | 3 +-- crates/ziggurat-zigbee/src/nwk/next_hop.rs | 12 ------------ 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/crates/ziggurat-phy-trel/src/lib.rs b/crates/ziggurat-phy-trel/src/lib.rs index 4a865c8..f96032f 100644 --- a/crates/ziggurat-phy-trel/src/lib.rs +++ b/crates/ziggurat-phy-trel/src/lib.rs @@ -1229,8 +1229,7 @@ impl Backbone { .peers .iter() .filter_map(|(eui64, p)| { - p.quic_port - .map(|qp| (*eui64, with_port(p.addr, qp))) + p.quic_port.map(|qp| (*eui64, with_port(p.addr, qp))) }) .collect(); for (peer, addr) in targets { diff --git a/crates/ziggurat-zigbee/src/nwk/next_hop.rs b/crates/ziggurat-zigbee/src/nwk/next_hop.rs index e1afcad..a98c251 100644 --- a/crates/ziggurat-zigbee/src/nwk/next_hop.rs +++ b/crates/ziggurat-zigbee/src/nwk/next_hop.rs @@ -109,18 +109,6 @@ pub fn resolve( return NextHop::Resolved(destination); } - // SPIKE (#84 bench, workaround for zigpy/ziggurat#88; real fix in #89): a router - // that just associated with us is one hop away, but - // has no route and no outgoing cost until its first Link Status (~15 s). ZBOSS - // routers unicast Node_Desc_req to the TC right after joining and leave when the - // reply (and APS ack) never arrives, so deliver to router children directly too. - if neighbors - .entries() - .any(|entry| entry.network_address == destination && entry.is_child()) - { - return NextHop::Resolved(destination); - } - // A stored source route (concentrator behavior) wins over the routing table. if policy.is_concentrator && !policy.force_route_discovery { match routing.route_to(destination, policy.max_source_route) {