diff --git a/crates/model/src/arch/nemotron_h_moe/host.rs b/crates/model/src/arch/nemotron_h_moe/host.rs new file mode 100644 index 000000000..0f644bc36 --- /dev/null +++ b/crates/model/src/arch/nemotron_h_moe/host.rs @@ -0,0 +1,45 @@ +//! The host tier's view of a Bolt layer's routed experts: relu² — the up and +//! down stacks, the layer's own expert count and width, read in the +//! `moe_latent_size` latent the experts run in. The one place that wires them +//! into a [`Relu2Spec`] (`HostLayer::build_relu2`); this architecture's files +//! hold no gate stack. + +use super::hparams::{Experts, Hparams}; +use crate::ModelError; +use crate::arch::names::routed_stacks; +use crate::moe::{HostLayer, Relu2Spec}; +use crate::r8file::R8Source; + +/// Layer `l`'s routed experts as the host tier serves them: ungated relu², +/// two stacks at the layer's own expert count and width in the latent width — +/// the MoE layers' shapes differ from layer to layer, so a caller serves each +/// layer through the layer its own shapes make. `None` for a layer that does +/// not route (a Mamba-2 or attention layer), an error for a layer past the +/// model. Load-time only: the stacks are checked here. +pub fn layer(src: R8Source<'_>, hp: &Hparams, l: usize) -> Result, ModelError> { + let Some(experts) = hp.experts.get(l) else { + return Err(ModelError::Shape { + what: "nemotron_h_moe host layer: the highest layer index, a layer inside the model's", + want_ne0: hp.experts.len().saturating_sub(1), + want_ne1: 1, + got_ne0: l, + got_ne1: 1, + }); + }; + let Some(Experts { n_expert, ff, .. }) = *experts else { + return Ok(None); + }; + // The common spelling of a routed stack; this architecture's files hold no gate. + let [_, up, down] = routed_stacks(l); + HostLayer::build_relu2( + src, + &Relu2Spec { + up: &up, + down: &down, + n_expert, + embd: hp.latent, + ff, + }, + ) + .map(Some) +} diff --git a/crates/model/src/arch/nemotron_h_moe/mod.rs b/crates/model/src/arch/nemotron_h_moe/mod.rs index e89a5e9aa..000fdaf6f 100644 --- a/crates/model/src/arch/nemotron_h_moe/mod.rs +++ b/crates/model/src/arch/nemotron_h_moe/mod.rs @@ -10,6 +10,7 @@ //! llama.cpp's `src/models/nemotron-h.cpp`, which loads both architectures. pub mod fixture; +pub mod host; pub mod hparams; pub mod names; pub mod place; diff --git a/crates/model/src/moe.rs b/crates/model/src/moe.rs index b24099830..6919d67d3 100644 --- a/crates/model/src/moe.rs +++ b/crates/model/src/moe.rs @@ -30,9 +30,9 @@ use gguf::{GgmlType, Gguf, Split, TensorInfo}; use crate::ModelError; use crate::ops::{ - self, DEFER_MAX_COLS, ExpertStack, GroupClaims, GroupInput, Lanes, R8Stack, RowLayout, - ShardTensor, StackTier, Tensor2, Tensor2View, UnionCall, UnionPlanView, UnionSlabs, UnionStack, - WarmSpan, Weight, defers, matmul_q, matmul_q_group, matmul_q_group_into_on, + self, DEFER_MAX_COLS, ExpertForm, ExpertStack, GroupClaims, GroupInput, Lanes, R8Stack, + RowLayout, ShardTensor, StackTier, Tensor2, Tensor2View, UnionCall, UnionPlanView, UnionSlabs, + UnionStack, WarmSpan, Weight, defers, matmul_q, matmul_q_group, matmul_q_group_into_on, matmul_q_group_swiglu, matmul_q_group_swiglu_into_on, }; use crate::profile; @@ -788,14 +788,14 @@ pub fn experts_into( let view = |views: &[TensorInfo], e: u32| -> Result, ModelError> { Weight::in_file(gguf, expert_of(views, e, plan.block)?) }; - let weights = |e: u32| -> Result<[Weight<'_>; 3], ModelError> { - Ok([ + let weights = |e: u32| -> Result, ModelError> { + Ok(ExpertLegs::Gated([ view(&plan.gate_views, e)?, view(&plan.up_views, e)?, view(&plan.down_views, e)?, - ]) + ])) }; - serve(weights, LegRun::flat(None), x, experts, out, scratch) + serve(weights, LegRun::flat(), x, experts, out, scratch) } /// Expert `e`'s view in one of a block plan's per-expert view lists; an id @@ -927,7 +927,8 @@ impl HostScratch { self.n_used } - /// The gate output of the last call's `i`-th expert. + /// The gate output of the last call's `i`-th expert (a relu² layer has + /// none: its slot holds no output of the call). pub fn gate(&self, i: usize) -> &Tensor2 { &self.gate_up[2 * i] } @@ -948,31 +949,41 @@ impl HostScratch { } } -/// How one host leg of one token runs: the experts' SwiGLU clamp (`None`: the -/// plain combine) and how the leg's rows are laid over the pool. +/// How one host leg of one token runs: the experts' form (which stacks the +/// leg reads and their combine) and how the leg's rows are laid over the +/// pool. #[derive(Clone, Copy)] struct LegRun { - limit: Option, + form: ExpertForm, lanes: Lanes, } impl LegRun { - /// The leg with rows in the pool's flat lanes. - fn flat(limit: Option) -> LegRun { + /// The gated leg without a clamp, rows in the pool's flat lanes. + fn flat() -> LegRun { LegRun { - limit, + form: ExpertForm::Gated { limit: 0.0 }, lanes: Lanes::Flat, } } } -/// The host leg of one token: each listed expert's `[gate, up, down]` from -/// `weights`, resolved in list order into the scratch's view lists, then +/// One expert's stacks as its layer's form reads them, resolved by the +/// caller from wherever the layer holds them. +enum ExpertLegs<'w> { + /// `[gate, up, down]`. + Gated([Weight<'w>; 3]), + /// The up and the down. + Relu2 { up: Weight<'w>, down: Weight<'w> }, +} + +/// The host leg of one token: each listed expert's stacks from `weights`, +/// resolved in list order into the scratch's view lists, then /// [`serve_resolved`]. The lists go back to the scratch, emptied, whether the /// call succeeds or not. The caller has checked the list against the /// scratch's width, so no push grows a list. fn serve<'w>( - weights: impl Fn(u32) -> Result<[Weight<'w>; 3], ModelError>, + weights: impl Fn(u32) -> Result, ModelError>, run: LegRun, x: &Tensor2, experts: &[(u32, f32)], @@ -984,9 +995,16 @@ fn serve<'w>( let r = experts .iter() .try_for_each(|&(e, _)| { - let [g, u, d] = weights(e)?; - gu.extend([g, u]); - down.push(d); + match weights(e)? { + ExpertLegs::Gated([g, u, d]) => { + gu.extend([g, u]); + down.push(d); + } + ExpertLegs::Relu2 { up, down: d } => { + gu.push(up); + down.push(d); + } + } Ok(()) }) .and_then(|()| serve_resolved(&gu, &down, run, x, experts, out, s)); @@ -995,15 +1013,16 @@ fn serve<'w>( r } -/// The host leg of one token over resolved weights: `gu` holds each listed -/// expert's gate and up interleaved, `down` its down. One group dispatch for -/// every gate and up (`x` quantized once), one for every down with each -/// combine — clamped when `limit` is given — produced inside it, then the +/// The host leg of one token over resolved weights: gated, `gu` holds each +/// listed expert's gate and up interleaved; a relu² leg holds its ups alone. +/// One group dispatch for the first stacks (`x` quantized once), one for +/// every down with each combine — the form's — produced inside it, then the /// weighted sum in list order into `out` (already zeroed). Profile level 1 /// records the leg in five parts that do not overlap: `host_x_quant` (x's -/// quantization), `host_gate_up` (the gate/up rows, per weight type), -/// `host_h_quant` (the combines and their quantization), `host_down` (the -/// down rows, per weight type) and `host_sum`. +/// quantization), `host_gate_up` (or `host_up`, a relu² leg's single stack; +/// the rows, per weight type), `host_h_quant` (the combines and their +/// quantization), `host_down` (the down rows, per weight type) and +/// `host_sum`. fn serve_resolved( gu: &[Weight<'_>], down: &[Weight<'_>], @@ -1023,22 +1042,31 @@ fn serve_resolved( srcs, .. } = s; + // A gated leg's first dispatch interleaves [gate, up] pairs — two blocks + // and two reads of `x` an expert; a relu² leg's holds the ups alone, and + // moves them to the up slots once the down has read them. + let (first, up_site) = match run.form { + ExpertForm::Gated { .. } => (2 * n, "host_gate_up"), + ExpertForm::Relu2 => (n, "host_up"), + }; let mut xl: Vec<&Tensor2> = relist(std::mem::take(xs)); - xl.extend(std::iter::repeat_n(x, 2 * n)); - let gt = matmul_q_group_into_on(run.lanes, "host_gate_up", gu, &xl, &mut gate_up[..2 * n]); + xl.extend(std::iter::repeat_n(x, first)); + let gt = matmul_q_group_into_on(run.lanes, up_site, gu, &xl, &mut gate_up[..first]); *xs = relist(xl); let gt = gt?; let mut sl: Vec> = relist(std::mem::take(srcs)); - sl.extend( - gate_up[..2 * n] - .as_chunks::<2>() - .0 - .iter() - .map(|[gate, up]| match run.limit { - Some(limit) => GroupInput::SwigluClamp(gate, up, limit), - None => GroupInput::Swiglu(gate, up), - }), - ); + match run.form { + ExpertForm::Gated { limit } => { + for [gate, up] in gate_up[..2 * n].as_chunks::<2>().0 { + sl.push(GroupInput::SwigluClamp(gate, up, limit)); + } + } + ExpertForm::Relu2 => { + for up in &gate_up[..n] { + sl.push(GroupInput::Relu2(up)); + } + } + } let dt = matmul_q_group_swiglu_into_on( run.lanes, "host_down", @@ -1049,6 +1077,13 @@ fn serve_resolved( ); *srcs = relist(sl); let dt = dt?; + if let ExpertForm::Relu2 = run.form { + // Back to back from the top: expert `i`'s up goes to slot `2 * i + 1`, + // where `HostScratch::up` reads it, and no block moves twice. + for i in (0..n).rev() { + gate_up.swap(i, 2 * i + 1); + } + } let t_sum = if lvl > 0 { Some(Instant::now()) } else { None }; for (&(_, w), d) in experts.iter().zip(&downs[..n]) { for (o, &dv) in out.iter_mut().zip(d.col(0)) { @@ -1539,7 +1574,11 @@ impl UnionScratch { /// The union call's shape conditions, each its own error: at most the /// scratch's columns, one list per column of `x`, `out` holding every -/// column's width, and the scratch made for the block's widths. +/// column's width, the scratch made for the block's width (`embd`), and its +/// expert width a capacity — the layer's `ff` at or below it, so one scratch +/// serves the run's layers of differing widths, and its combine slab (sized +/// by [`ops::col_bytes_max`] of the capacity) holding the layer's down's +/// columns. fn check_union_call( x: Tensor2View<'_>, lists: &[&[(u32, f32)]], @@ -1575,9 +1614,19 @@ fn check_union_call( got_ne1: 1, }); } - if (scratch.embd, scratch.ff) != (embd, ff) { + if scratch.embd != embd { return Err(ModelError::Shape { - what: "host union: the scratch must be made for the block's widths", + what: "host union: the scratch must be made for the block's width", + want_ne0: embd, + want_ne1: ff, + got_ne0: scratch.embd, + got_ne1: scratch.ff, + }); + } + if ff > scratch.ff || ops::col_bytes_max(ff) > ops::col_bytes_max(scratch.ff) { + return Err(ModelError::Shape { + what: "host union: the scratch must be made for the block's widths (its expert \ + width is a capacity: the layer's ff at or below it, with room for its column)", want_ne0: embd, want_ne1: ff, got_ne0: scratch.embd, @@ -1705,7 +1754,7 @@ fn serve_tiered<'w>( assert!(batch > 0, "host union: a batch holds at least one expert"); // Refused before any claim: an expert past the stacks' (the passes' own // check, made once here). - UnionCall::new(s.plan.view(), stacks, x, run.limit, s.ff)?; + UnionCall::new(s.plan.view(), stacks, x, run.form)?; let mut stacks = stacks; tier.lend(&mut stacks); s.tier_ids.clear(); @@ -1762,7 +1811,6 @@ fn serve_batched<'w>( ) -> Result<(), ModelError> { let Batching { tier, keep } = batching; let (batch, embd) = (tier.batch, x.ne0()); - let ff = s.ff; let UnionScratch { plan, xq, @@ -1788,7 +1836,7 @@ fn serve_batched<'w>( let from = g.checked_sub(1).map_or(0, |p| ends[p]); from..ends[g] }; - let whole = UnionCall::new(plan.view(), stacks, x, run.limit, ff)?.spread_over(run.lanes); + let whole = UnionCall::new(plan.view(), stacks, x, run.form)?.spread_over(run.lanes); let mut xqs = whole.quantize_batched(xq, xq_up, passes); slot.clear(); slot.resize(plan.n(), 0); @@ -1823,8 +1871,7 @@ fn serve_batched<'w>( for (i, &d) in order[span(g)].iter().enumerate() { slot[d as usize] = base + sub.off[i]; } - let batch_call = - UnionCall::new(sub.view(), stacks, x, run.limit, ff)?.spread_over(run.lanes); + let batch_call = UnionCall::new(sub.view(), stacks, x, run.form)?.spread_over(run.lanes); batch_call.run_batch(&mut xqs, gu, qc, &mut store[base * embd..], None, passes)?; base += sub.cols.len(); drop(pin); @@ -1871,7 +1918,7 @@ fn serve_step_split<'w>( drop(pin); return served; } - let (embd, ff) = (x.ne0(), s.ff); + let embd = x.ne0(); let UnionScratch { plan, xq, @@ -1893,7 +1940,7 @@ fn serve_step_split<'w>( .. } = batches; let (order, ids) = (&*order, &*ids); - let whole = UnionCall::new(plan.view(), stacks, x, run.limit, ff)?.spread_over(run.lanes); + let whole = UnionCall::new(plan.view(), stacks, x, run.form)?.spread_over(run.lanes); let deferred = defers(x.ne1()); let mut xqs = if deferred { whole.x_rooms(xq, xq_up) @@ -1925,7 +1972,7 @@ fn serve_step_split<'w>( for (i, &d) in order[span].iter().enumerate() { slot[d as usize] = base + sub.off[i]; } - let call = UnionCall::new(sub.view(), stacks, x, run.limit, ff)?.spread_over(run.lanes); + let call = UnionCall::new(sub.view(), stacks, x, run.form)?.spread_over(run.lanes); let claim = deferred.then(|| GroupClaims { x: g == 0, states: &mut claims[..], @@ -1960,7 +2007,7 @@ fn serve_planned( out.fill(0.0); return Ok(()); } - let call = UnionCall::new(s.plan.view(), stacks, x, run.limit, s.ff)?.spread_over(run.lanes); + let call = UnionCall::new(s.plan.view(), stacks, x, run.form)?.spread_over(run.lanes); let slabs = UnionSlabs { xq: &mut s.xq, xq_up: &mut s.xq_up, @@ -2002,13 +2049,15 @@ pub fn experts_union_into<'x>( ExpertStack::in_file(gguf, up)?, ExpertStack::in_file(gguf, down)?, ]; - serve_union(stacks, LegRun::flat(None), x, lists, out, scratch, None) + serve_union(stacks, LegRun::flat(), x, lists, out, scratch, None) } /// What a host tier needs to serve one layer's routed experts, in the /// file's own terms: the three stacks' tensor names, the expert count, the -/// widths and the routed experts' SwiGLU limit. The architecture fills it; -/// this module reads no model's names or keys. +/// widths and the routed experts' SwiGLU limit — the gated reading of a +/// layer ([`ExpertForm::Gated`], which [`HostLayer::build`] serves it by); +/// the relu² reading has its own spec ([`Relu2Spec`], [`HostLayer::build_relu2`]). +/// The architecture fills it; this module reads no model's names or keys. pub struct HostLayerSpec<'a> { pub gate: &'a str, pub up: &'a str, @@ -2021,6 +2070,18 @@ pub struct HostLayerSpec<'a> { pub swiglu_limit: f32, } +/// What a host tier needs to serve one relu² layer's routed experts +/// ([`ExpertForm::Relu2`]): the two stacks' tensor names, the expert count and +/// the widths — `embd` the width the experts run in, `ff` this layer's own. +/// The architecture fills it; there is no gate name to give. +pub struct Relu2Spec<'a> { + pub up: &'a str, + pub down: &'a str, + pub n_expert: usize, + pub embd: usize, + pub ff: usize, +} + /// The host tier's matmul path for each (weight type, row width, row /// layout) it serves, printed once per process when a layer first brings it /// in: `load host_tier type=iq3_xxs k=4096 path=fused` — qdot's fused @@ -2130,6 +2191,13 @@ pub enum TierError { ids: usize, slots: usize, }, + /// A relu² layer met the NVMe tier: the arena lends each expert's bytes + /// as `[gate, up, down]` parts and its paths lay them so, and a form + /// that reads two stacks has no gate part for the third. + #[error( + "layer {layer}'s NVMe tier serves gated experts only: a relu² layer holds no gate part" + )] + Relu2Tiered { layer: usize }, /// A batch pin was refused: the tier's own error, naming what it /// pinned. #[error("layer {layer}: pinning a batch of experts: {source}")] @@ -2463,17 +2531,27 @@ impl LayerTier { } } -/// One layer's routed experts as a host tier serves them: the three stacks — -/// the gate and the up from the file or from its r8 sidecar, the down from -/// the file — and the SwiGLU limit. Built once at load -/// ([`HostLayer::build`]); a call cuts its experts' matrices from the stacks, -/// so a layer holds three headers, not one view per expert. +/// The tensor names a layer build reads; a relu² layer names no gate. +struct StackNames<'a> { + gate: Option<&'a str>, + up: &'a str, + down: &'a str, +} + +/// One layer's routed experts as a host tier serves them: its form +/// ([`ExpertForm`]) and the stacks that form reads — gated, the gate and the +/// up from the file or from its r8 sidecar and the down from the file; a +/// relu² layer, the up and the down, no gate. Built once at load +/// ([`HostLayer::build`], [`HostLayer::build_relu2`]); a call cuts its +/// experts' matrices from the stacks, so a layer holds headers, not one view +/// per expert. pub struct HostLayer { - gate: GateUp, + /// The gate stack — `None` on a relu² layer, which holds none. + gate: Option, up: GateUp, down: ShardTensor, n_expert: usize, - limit: f32, + form: ExpertForm, tier: Option, } @@ -2493,51 +2571,103 @@ impl HostLayer { /// not a property of the layer's tensors, so it rides beside the spec, in /// the pair. pub fn build(src: R8Source<'_>, spec: &HostLayerSpec<'_>) -> Result { + Self::build_form( + src, + StackNames { + gate: Some(spec.gate), + up: spec.up, + down: spec.down, + }, + (spec.n_expert, spec.embd, spec.ff), + ExpertForm::Gated { + limit: spec.swiglu_limit, + }, + ) + } + + /// [`HostLayer::build`] for a relu² layer ([`Relu2Spec`]): the up and the + /// down checked as the gated build checks them, no gate stack looked for. + /// The form rides beside the spec like the r8 sidecar does: an activation + /// fact the load resolves, not a property of the layer's tensors. + pub fn build_relu2(src: R8Source<'_>, spec: &Relu2Spec<'_>) -> Result { + Self::build_form( + src, + StackNames { + gate: None, + up: spec.up, + down: spec.down, + }, + (spec.n_expert, spec.embd, spec.ff), + ExpertForm::Relu2, + ) + } + + /// [`HostLayer::build`] and [`HostLayer::build_relu2`]'s one body over + /// the named stacks: a gate name (`Some`) is a stack the layer holds, found, + /// checked and served; `None`, one it does not. + fn build_form( + src: R8Source<'_>, + names: StackNames<'_>, + (n_expert, embd, ff): (usize, usize, usize), + form: ExpertForm, + ) -> Result { + let StackNames { gate, up, down } = names; let split = src.split(); - let gate = ShardTensor::find(split, spec.gate)?; - let up = ShardTensor::find(split, spec.up)?; - let down = ShardTensor::find(split, spec.down)?; - for (t, k, n) in [ - (&gate, spec.embd, spec.ff), - (&up, spec.embd, spec.ff), - (&down, spec.ff, spec.embd), - ] { - expect_stack(t.info(), k, n, spec.n_expert)?; + let found = gate.map(|g| ShardTensor::find(split, g)).transpose()?; + let up_stack = ShardTensor::find(split, up)?; + let down_stack = ShardTensor::find(split, down)?; + if let Some(gate) = &found { + expect_stack(gate.info(), embd, ff, n_expert)?; // The per-expert cut: `expert_view`'s check, once, here. - expert_view(t.info(), 0, spec.n_expert)?; + expert_view(gate.info(), 0, n_expert)?; + } + for (t, k, n) in [(&up_stack, embd, ff), (&down_stack, ff, embd)] { + expect_stack(t.info(), k, n, n_expert)?; + // The per-expert cut: `expert_view`'s check, once, here. + expert_view(t.info(), 0, n_expert)?; } let (gate, up) = match src.sidecar() { - None => (GateUp::File(gate), GateUp::File(up)), + None => (found.map(GateUp::File), GateUp::File(up_stack)), Some(side) => { - if side.find(spec.down).is_some() { + if side.find(down).is_some() { return Err(R8Error::HoldsDown { path: side.path().to_path_buf(), - tensor: spec.down.to_string(), + tensor: down.to_string(), } .into()); } - ( - GateUp::R8(R8Stack::of(src, spec.gate)?), - GateUp::R8(R8Stack::of(src, spec.up)?), - ) + let gate = gate + .map(|g| R8Stack::of(src, g).map(GateUp::R8)) + .transpose()?; + (gate, GateUp::R8(R8Stack::of(src, up)?)) } }; + // A relu² layer's stacks as the union reads them: the up again in the + // gate's slot, one stack where the gated form reads two. + let gate_stack = || match &gate { + Some(g) => g.stack(split), + None => up.stack(split), + }; UnionStack::of_call( - &[gate.stack(split)?, up.stack(split)?, down.stack(split)?], - spec.embd, - spec.ff, + &[gate_stack()?, up.stack(split)?, down_stack.stack(split)?], + embd, + ff, )?; - announce_paths(&[ - (gate.source().info().ty, spec.embd, gate.layout()), - (up.source().info().ty, spec.embd, up.layout()), - (down.info().ty, spec.ff, RowLayout::Rows), + let mut paths = Vec::with_capacity(3); + if let Some(gate) = &gate { + paths.push((gate.source().info().ty, embd, gate.layout())); + } + paths.extend([ + (up.source().info().ty, embd, up.layout()), + (down_stack.info().ty, ff, RowLayout::Rows), ]); + announce_paths(&paths); Ok(HostLayer { gate, up, - down, - n_expert: spec.n_expert, - limit: spec.swiglu_limit, + down: down_stack, + n_expert, + form, tier: None, }) } @@ -2593,18 +2723,34 @@ impl HostLayer { /// The three stacks, `[gate, up, down]`, as the source file holds them: /// for a gate or an up read from the r8 sidecar, the source stack its copy - /// was checked against ([`HostLayer::layouts`] tells which). + /// was checked against ([`HostLayer::layouts`] tells which). A relu² + /// layer's gate slot holds its up again — the file holds no gate, and the + /// form ([`HostLayer::form`]) is the one owner of which stacks a call + /// reads. pub fn stacks(&self) -> [&ShardTensor; 3] { - [self.gate.source(), self.up.source(), &self.down] + [ + self.gate.as_ref().map_or(self.up.source(), GateUp::source), + self.up.source(), + &self.down, + ] } /// Where each stack's rows lie, `[gate, up, down]` — every expert of a /// stack has its stack's layout: [`RowLayout::R8`] for a stack the layer - /// reads from the r8 sidecar, which no path reads as Q3_K rows. What the - /// load's reading made of the layer, for a caller that holds it to that - /// reading. + /// reads from the r8 sidecar, which no path reads as Q3_K rows. A relu² + /// layer's gate slot carries its up's layout. What the load's reading + /// made of the layer, for a caller that holds it to that reading. pub fn layouts(&self) -> [RowLayout; 3] { - [self.gate.layout(), self.up.layout(), RowLayout::Rows] + [ + self.gate.as_ref().map_or(self.up.layout(), GateUp::layout), + self.up.layout(), + RowLayout::Rows, + ] + } + + /// The layer's expert form. + pub fn form(&self) -> ExpertForm { + self.form } /// `src` is a pair this layer reads beside: one that reads the open of @@ -2612,7 +2758,8 @@ impl HostLayer { /// pair's own check having run when it was made. A layer that reads only /// the file takes any pair: it reads no sidecar. fn read_beside(&self, src: R8Source<'_>) -> Result<(), ModelError> { - for s in [&self.gate, &self.up] { + let ups = [self.gate.as_ref(), Some(&self.up)]; + for s in ups.into_iter().flatten() { if let GateUp::R8(s) = s { src.reads(s.sidecar(), "a host layer's call")?; } @@ -2620,31 +2767,39 @@ impl HostLayer { Ok(()) } - /// The three stacks as a union call reads them, resolved once. + /// The stacks as a union call reads them, `[gate, up, down]`: a relu² + /// layer's up again in the gate's slot, one stack where the gated form + /// reads two. fn stacks_of<'a>(&'a self, split: &'a Split) -> Result<[ExpertStack<'a>; 3], ModelError> { - Ok([ - self.gate.stack(split)?, - self.up.stack(split)?, - self.down.stack(split)?, - ]) + let gate = match &self.gate { + Some(g) => g.stack(split)?, + None => self.up.stack(split)?, + }; + Ok([gate, self.up.stack(split)?, self.down.stack(split)?]) } pub fn n_expert(&self) -> usize { self.n_expert } + /// The routed experts' SwiGLU limit, a gated layer's; `0.0` for a relu² + /// layer, which clamps nothing. pub fn swiglu_limit(&self) -> f32 { - self.limit + match self.form { + ExpertForm::Gated { limit } => limit, + ExpertForm::Relu2 => 0.0, + } } /// [`experts_into`] for this layer: a given list of routed experts of - /// ONE token, `out = Σ_i w_i · down_{e_i}(clamp(up_{e_i}·x, ±L) ⊙ - /// min(silu(gate_{e_i}·x), L))` in list order, each matrix read from - /// the shard that holds it, or the sidecar. `src` is the pair the layer - /// was built from. `scratch` is the caller's, made for this layer's - /// widths and the model's routed width, and a list past that width is - /// refused by name; after the call it holds what the call computed. The - /// leg's two row dispatches are laid over the pool's CCDs + /// ONE token — `out = Σ_i w_i · down_{e_i}(clamp(up_{e_i}·x, ±L) ⊙ + /// min(silu(gate_{e_i}·x), L))` gated, `out = Σ_i w_i · + /// down_{e_i}(relu(up_{e_i}·x)²)` for a relu² layer — in list order, + /// each matrix read from the shard that holds it, or the sidecar. `src` + /// is the pair the layer was built from. `scratch` is the caller's, made + /// for this layer's widths and the model's routed width, and a list past + /// that width is refused by name; after the call it holds what the call + /// computed. The leg's two row dispatches are laid over the pool's CCDs /// ([`Lanes::host_tier`]), which changes no value. pub fn experts_into( &self, @@ -2707,6 +2862,7 @@ impl HostLayer { self.read_beside(src)?; let split = src.split(); check_host_call(x, experts, out, scratch.n_used)?; + self.refuse_relu2_tier()?; if self.tier.is_some() && let Some(union) = union { @@ -2729,6 +2885,20 @@ impl HostLayer { self.serve_column(lanes, split, x, experts, out, scratch) } + /// A relu² layer's refusal of the NVMe tier, at every entry that could + /// reach one: the arena lends each expert's bytes as `[gate, up, down]` + /// parts and its paths lay them so, and a form that reads two stacks has + /// no gate part for the third ([`HostLayer::attach_tier`] cannot refuse — + /// the gpu side's run build calls it, and that side holds no form yet). + fn refuse_relu2_tier(&self) -> Result<(), ModelError> { + if let ExpertForm::Relu2 = self.form + && let Some(tier) = &self.tier + { + return Err(TierError::Relu2Tiered { layer: tier.layer }.into()); + } + Ok(()) + } + /// Today's one-column leg: `out` is the list-order sum, each expert read /// from the mapping or from a slot the caller already pinned. fn serve_column( @@ -2744,21 +2914,30 @@ impl HostLayer { if experts.is_empty() { return Ok(()); } - let weights = |e: u32| -> Result<[Weight<'_>; 3], ModelError> { - Ok([ - self.tier_or_mapping(e, 0, self.gate.source(), self.gate.layout(), || { - self.gate.expert(split, e as usize) - })?, - self.tier_or_mapping(e, 1, self.up.source(), self.up.layout(), || { - self.up.expert(split, e as usize) - })?, - self.tier_or_mapping(e, 2, &self.down, RowLayout::Rows, || { - self.down.expert(split, e as usize) - })?, - ]) + let weights = |e: u32| -> Result, ModelError> { + let down = self.tier_or_mapping(e, 2, &self.down, RowLayout::Rows, || { + self.down.expert(split, e as usize) + })?; + match &self.gate { + Some(gate) => Ok(ExpertLegs::Gated([ + self.tier_or_mapping(e, 0, gate.source(), gate.layout(), || { + gate.expert(split, e as usize) + })?, + self.tier_or_mapping(e, 1, self.up.source(), self.up.layout(), || { + self.up.expert(split, e as usize) + })?, + down, + ])), + None => Ok(ExpertLegs::Relu2 { + up: self.tier_or_mapping(e, 1, self.up.source(), self.up.layout(), || { + self.up.expert(split, e as usize) + })?, + down, + }), + } }; let run = LegRun { - limit: Some(self.limit), + form: self.form, lanes, }; serve(weights, run, x, experts, out, scratch) @@ -2779,7 +2958,7 @@ impl HostLayer { { return Err(ModelError::MissingTensor(format!( "expert {e} of {}", - self.gate.source().info().name + self.up.source().info().name ))); } s.tier_ids.clear(); @@ -2826,7 +3005,7 @@ impl HostLayer { let mut stacks = self.stacks_of(split)?; tier.lend(&mut stacks); let run = LegRun { - limit: Some(self.limit), + form: self.form, lanes, }; serve_step_split(stacks, run, x.view(), &lists, out, union, tier, ticket) @@ -2869,17 +3048,18 @@ impl HostLayer { scratch: &mut UnionScratch, ) -> Result<(), ModelError> { self.read_beside(src)?; + self.refuse_relu2_tier()?; let split = src.split(); let x = x.into(); check_union_call( x, lists, out, - self.gate.source().info().dims[1] as usize, + self.up.source().info().dims[1] as usize, scratch, )?; let run = LegRun { - limit: Some(self.limit), + form: self.form, lanes, }; serve_union( @@ -2936,11 +3116,12 @@ impl HostLayer { self.read_beside(src)?; let split = src.split(); let x = x.into(); + self.refuse_relu2_tier()?; check_union_call( x, lists, out, - self.gate.source().info().dims[1] as usize, + self.up.source().info().dims[1] as usize, scratch, )?; // The plan's ids are the call's distinct ids, ascending; the union @@ -2948,7 +3129,7 @@ impl HostLayer { self.prepare_step_ids(tier, lists, scratch)?; let mut stacks = self.stacks_of(split)?; let run = LegRun { - limit: Some(self.limit), + form: self.form, lanes, }; if scratch.tier_ids.is_empty() { @@ -2972,10 +3153,11 @@ impl HostLayer { } /// The matrices a warm routine fills for `ids`, pushed to `out` in `ids`' - /// order, each id's gate, up and down as the leg reads them - /// ([`HostLayer::experts_into`], [`HostLayer::experts_step_union_into`]): - /// from the file mapping or the r8 sidecar, or — an id outside the plan's - /// host segment — from its arena slot. An id whose slot is unfilled is + /// order, each id's stacks as the leg reads them ([`HostLayer::experts_into`], + /// [`HostLayer::experts_step_union_into`]) — the gate, the up and the down + /// gated; the up and the down for a relu² layer — from the file mapping or + /// the r8 sidecar, or — an id outside the plan's host segment — from its + /// arena slot. An id whose slot is unfilled is /// left out whole: a warm never fills a slot ([`TierSlots::ensure`]). /// Each span carries the units of the step legs' pass that reads it, so /// the CCD that computes a row is the CCD that warms it @@ -3007,15 +3189,22 @@ impl HostLayer { if let Some(tier) = &self.tier { tier.lend(&mut stacks); } + // A relu² layer warms its up and its down; the stacks' gate slot is + // the up again. + let warmed = match self.form { + ExpertForm::Gated { .. } => &stacks[..], + ExpertForm::Relu2 => &stacks[1..], + }; 'ids: for &id in ids { if id as usize >= self.n_expert { return Err(ModelError::MissingTensor(format!( "expert {id} of {}", - self.gate.source().info().name + self.up.source().info().name ))); } - let mut parts = [None; 3]; - for (part, stack) in stacks.iter().enumerate() { + let mut all: [Option>; 3] = [None; 3]; + let parts = &mut all[..warmed.len()]; + for (part, stack) in warmed.iter().enumerate() { let Some(span) = stack.warm_span(id) else { continue 'ids; }; @@ -3031,7 +3220,7 @@ impl HostLayer { } parts[part] = Some(span); } - out.extend(parts.into_iter().flatten()); + out.extend(parts.iter().copied().flatten()); } Ok(()) } @@ -3040,10 +3229,10 @@ impl HostLayer { #[cfg(test)] mod tests { use super::{ - Buckets, HostLayer, HostLayerSpec, HostScratch, TOUCHED, UnionScratch, check_host_call, - gather_expert_inputs, last_touched_experts, relist, reset_touched, + Buckets, HostLayer, HostLayerSpec, HostScratch, Relu2Spec, TOUCHED, UnionScratch, + check_host_call, gather_expert_inputs, last_touched_experts, relist, reset_touched, }; - use crate::ops::{self, RowLayout, Tensor2}; + use crate::ops::{self, ExpertForm, RowLayout, Tensor2}; use crate::r8file::{HostR8, R8Source, Sidecar}; use gguf::GgmlType; use gguf::write::{Layout, TensorDecl, Writer}; @@ -3272,6 +3461,85 @@ mod tests { } } + /// A relu² layer's build, by name: a spec whose stacks carry other dims + /// than it says is refused with the stack's own error; a relu² layer that + /// built reads its up again in the stacks' gate slot — the file's gate + /// stack, which it holds here, is never read — and carries the relu² + /// form; and a relu² layer with the NVMe tier attached is refused by name + /// at every entry that could reach one. + #[test] + fn host_layer_build_relu2_refuses_by_name() { + let (embd, ff, n_expert) = (256, 512, 4); + let check = |got: String, want: &str| { + assert!( + got.contains(want), + "refusal must name {want:?}, got {got:?}" + ); + println!("relu2 refusal {want:?}: {got}"); + }; + let spec = |embd, ff| Relu2Spec { + up: "up_exps", + down: "down_exps", + n_expert, + embd, + ff, + }; + let path = layer_file("relu2", [GgmlType::Q4_K; 3], embd, ff, n_expert); + let split = gguf::Split::open(&path).unwrap(); + let src = R8Source::rows(&split); + check( + HostLayer::build_relu2(src, &spec(ff, embd)) + .err() + .expect("a spec of other dims is refused") + .to_string(), + "expert stack: expected [512, 256], got [256, 512]", + ); + let mut layer = + HostLayer::build_relu2(src, &spec(embd, ff)).expect("two stacks of Q4_K build"); + assert_eq!( + layer.form(), + ExpertForm::Relu2, + "the layer carries its form" + ); + assert_eq!( + ( + &layer.stacks()[0].info().name, + &layer.stacks()[1].info().name + ), + (&"up_exps".to_string(), &"up_exps".to_string()), + "the gate slot holds the up again" + ); + + let mut spans = Vec::new(); + layer.warm_spans(src, &[0], &mut spans).unwrap(); + assert_eq!(spans.len(), 2, "a relu² layer warms its up and its down"); + + // The NVMe tier: refused by name at the one-column, the union and the + // step-union entries, before any slot is asked for. + let tier = std::sync::Arc::new(MockTier::of(&split, &layer, false)); + layer.attach_tier(4, 1..2, tier); + let (x, list) = (Tensor2::zeros(embd, 1), vec![(0u32, 1.0f32)]); + let mut out = vec![0.0f32; embd]; + let mut host = HostScratch::new(embd, ff, n_expert).unwrap(); + let e = layer + .experts_into(src, &x, &list, &mut out, &mut host) + .expect_err("a relu² layer refuses the tier") + .to_string(); + assert!(e.contains("gated experts only"), "{e}"); + let mut us = UnionScratch::new_routed(embd, ff, 4, n_expert).unwrap(); + let e = layer + .experts_union_into(src, &x, &[&list[..]], &mut out, &mut us) + .expect_err("a relu² layer refuses the tier at the union") + .to_string(); + assert!(e.contains("gated experts only"), "{e}"); + let e = layer + .experts_step_union_into(src, &x, &[&list[..]], &mut out, &mut us) + .expect_err("a relu² layer refuses the tier at the step union") + .to_string(); + assert!(e.contains("gated experts only"), "{e}"); + std::fs::remove_file(&path).unwrap(); + } + /// A `BLOOMERY_QDOT_AVX512` value qdot refuses stops the host tier's load: /// in a fresh process with the lever at `2`, building a Q4_K layer panics /// naming the lever ([`qdot::q4k_avx512`], which [`ops::host_stack_runs`] diff --git a/crates/model/src/ops.rs b/crates/model/src/ops.rs index f473774bd..abac36c84 100644 --- a/crates/model/src/ops.rs +++ b/crates/model/src/ops.rs @@ -670,6 +670,54 @@ pub enum GroupInput<'a> { /// feeds the pair's weight ([`qdot::swiglu_clamp`]; a limit at or below /// 1e-6 clamps nothing). SwigluClamp(&'a Tensor2, &'a Tensor2, f32), + /// (`up`) — `relu(up)²` feeds the pair's weight: the ungated expert form + /// ([`ExpertForm::Relu2`]), whose layer holds no gate stack. + Relu2(&'a Tensor2), +} + +/// The combine a [`GroupInput::Swiglu`], [`GroupInput::SwigluClamp`] or +/// [`GroupInput::Relu2`] input stands for: the blocks it reads and the one +/// call every path makes of them ([`InputCombine::write_par`]). +#[derive(Clone, Copy)] +enum InputCombine<'a> { + /// `silu(gate) · up`, clamped with `limit`. + Swiglu { + gate: &'a Tensor2, + up: &'a Tensor2, + limit: Option, + }, + /// `relu(up)²`. + Relu2 { up: &'a Tensor2 }, +} + +impl InputCombine<'_> { + /// The combine of every column, into `out`. + fn write_par(&self, out: &mut [f32]) { + match self { + InputCombine::Swiglu { gate, up, limit } => { + swiglu_into(&gate.data, &up.data, *limit, out) + } + InputCombine::Relu2 { up } => relu2_into(&up.data, out), + } + } + + /// The combine of column `t` alone, into `out`. + fn col_into(&self, t: usize, out: &mut [f32]) { + match self { + InputCombine::Swiglu { gate, up, limit } => { + swiglu_into(gate.col(t), up.col(t), *limit, out) + } + InputCombine::Relu2 { up } => relu2_into(up.col(t), out), + } + } + + /// The profiler site the combine's time records under. + fn site(&self) -> &'static str { + match self { + InputCombine::Swiglu { .. } => "swiglu", + InputCombine::Relu2 { .. } => "relu2", + } + } } impl<'a> GroupInput<'a> { @@ -680,6 +728,7 @@ impl<'a> GroupInput<'a> { GroupInput::Cols(x, _) | GroupInput::Quantized(x, _, _) => x, GroupInput::Ready(x) => x.view(), GroupInput::Swiglu(gate, _) | GroupInput::SwigluClamp(gate, _, _) => gate.view(), + GroupInput::Relu2(up) => up.view(), } } @@ -697,13 +746,21 @@ impl<'a> GroupInput<'a> { self.cols().map_or(self.shape_of().ne1(), <[usize]>::len) } - /// The combine this input stands for — gate, up and the clamp limit — or - /// `None` for a ready block. - fn combine(self) -> Option<(&'a Tensor2, &'a Tensor2, Option)> { + /// The combine this input stands for, or `None` for a ready block. + fn combine(self) -> Option> { match self { GroupInput::Ready(_) | GroupInput::Cols(..) | GroupInput::Quantized(..) => None, - GroupInput::Swiglu(gate, up) => Some((gate, up, None)), - GroupInput::SwigluClamp(gate, up, limit) => Some((gate, up, Some(limit))), + GroupInput::Swiglu(gate, up) => Some(InputCombine::Swiglu { + gate, + up, + limit: None, + }), + GroupInput::SwigluClamp(gate, up, limit) => Some(InputCombine::Swiglu { + gate, + up, + limit: Some(limit), + }), + GroupInput::Relu2(up) => Some(InputCombine::Relu2 { up }), } } } @@ -1570,7 +1627,7 @@ enum Dest<'o> { } /// `silu(gate) · up` into `out`, clamped when the pair carries a limit — the -/// one call every combine makes, on a claimant or on the caller. +/// one call every gated combine makes, on a claimant or on the caller. fn swiglu_into(gate: &[f32], up: &[f32], limit: Option, out: &mut [f32]) { match limit { None => qdot::swiglu(gate, up, out), @@ -1578,21 +1635,48 @@ fn swiglu_into(gate: &[f32], up: &[f32], limit: Option, out: &mut [f32]) { } } +/// `relu(up)²` into `out`: `r = max(up[i], 0)` then `r · r` of the f32, +/// written once — the one call every relu² combine makes, on a claimant or on +/// the caller. A NaN or an infinity in `up` is refused by name after the loop +/// (the `max` would turn a NaN, and a negative infinity, into zero before the +/// quantizer could see it); the loop itself carries no branch. +fn relu2_into(up: &[f32], out: &mut [f32]) { + let mut invalid = false; + for (o, &u) in out.iter_mut().zip(up) { + invalid |= !u.is_finite(); + let r = u.max(0.0); + *o = r * r; + } + if invalid { + let bad = up.iter().find(|u| !u.is_finite()); + panic!("relu2: non-finite activation value {bad:?}"); + } +} + /// The allocating entry's caller-side combine: a fresh block and its time, -/// recorded under the `swiglu` site. A plain pair is `ffn::swiglu_timed` +/// recorded under the combine's site. A plain pair is `ffn::swiglu_timed` /// itself; a clamped one is the same shape with the clamped call. -fn combine_timed(gate: &Tensor2, up: &Tensor2, limit: Option) -> (Tensor2, u64) { - if limit.is_none() { +fn combine_timed(combine: InputCombine<'_>) -> (Tensor2, u64) { + if let InputCombine::Swiglu { + gate, + up, + limit: None, + } = combine + { return swiglu_timed(gate, up); } let lvl = profile::level(); let t_call = if lvl > 0 { Some(Instant::now()) } else { None }; - let mut out = Tensor2::scratch(gate.ne0, gate.ne1); - swiglu_into(&gate.data, &up.data, limit, &mut out.data); + let (rows, cols) = match combine { + InputCombine::Swiglu { gate, .. } => (gate.ne0, gate.ne1), + InputCombine::Relu2 { up } => (up.ne0, up.ne1), + }; + let mut out = Tensor2::scratch(rows, cols); + combine.write_par(&mut out.data); let ns = match t_call { Some(t_call) => { let ns = t_call.elapsed().as_nanos() as u64; - profile::record_time("swiglu", ns); + profile::record_time(combine.site(), ns); ns } None => 0, @@ -1635,6 +1719,11 @@ fn group_core( let n_combine = (0..n_pairs) .filter(|&i| inputs.get(i).combine().is_some()) .count(); + // One combine form a group (one leg, one form): the site its deferred + // claim time records under. + let combine_site = (0..n_pairs) + .find_map(|i| inputs.get(i).combine().map(|c| c.site())) + .unwrap_or("swiglu"); let defer = match inputs { Inputs::Ready(_) => defer_quant(), Inputs::Mixed(srcs) => { @@ -1741,8 +1830,10 @@ fn group_core( // into the pair's par. let (x, swiglu) = match src.combine() { None => (SlotBlock::Input(xin), None), - Some((gate, up, limit)) => { - if (up.ne0, up.ne1) != (gate.ne0, gate.ne1) { + Some(combine) => { + if let InputCombine::Swiglu { gate, up, .. } = combine + && (up.ne0, up.ne1) != (gate.ne0, gate.ne1) + { return Err(crate::ModelError::Shape { what: "swiglu up", want_ne0: gate.ne0, @@ -1754,7 +1845,7 @@ fn group_core( let blk: *mut Tensor2 = match &mut sink { Sink::Alloc { pars, .. } => { let par = if caller_combines { - let (par, ns) = combine_timed(gate, up, limit); + let (par, ns) = combine_timed(combine); caller_combine_ns += ns; par } else { @@ -1784,7 +1875,7 @@ fn group_core( if caller_combines { let t_c = if lvl > 0 { Some(Instant::now()) } else { None }; // SAFETY: as the read above. - swiglu_into(&gate.data, &up.data, limit, unsafe { &mut (*blk).data }); + combine.write_par(unsafe { &mut (*blk).data }); if let Some(t_c) = t_c { caller_combine_ns += t_c.elapsed().as_nanos() as u64; } @@ -1807,9 +1898,7 @@ fn group_core( ( SlotBlock::Par(par), Some(Combine { - gate, - up, - limit, + combine, par: write, }), ) @@ -1950,7 +2039,7 @@ fn group_core( caller_combine_ns + rt.stage_ns } else { if defer && lvl > 0 && n_combine > 0 { - profile::record_time("swiglu", rt.swiglu_ns); + profile::record_time(combine_site, rt.swiglu_ns); } caller_combine_ns + rt.swiglu_ns }; @@ -2649,13 +2738,11 @@ impl<'a> SlotBlock<'a> { } } -/// A combine slot's producer: gate, up, the clamp limit when the pair -/// carries one, and the write view of the par block the slot's `x` names. +/// A combine slot's producer: the combine the slot's input stands for and the +/// write view of the par block the slot's `x` names. #[derive(Clone, Copy)] struct Combine<'a> { - gate: &'a Tensor2, - up: &'a Tensor2, - limit: Option, + combine: InputCombine<'a>, par: ParWrite, } @@ -2707,7 +2794,7 @@ unsafe fn quant_range( // `# Safety`); nothing reads the block before the join. let par = unsafe { std::slice::from_raw_parts_mut(comb.par.0.add(t * s.k), s.k) }; - swiglu_into(comb.gate.col(t), comb.up.col(t), comb.limit, par); + comb.combine.col_into(t, par); // SAFETY: column `t` of slot `d` is inside `cols`, this caller's alone (this fn's `# Safety`). unsafe { shared.get(d).quantize_into(s.ty, s.k, par, t) }; } @@ -2904,7 +2991,7 @@ impl<'a> DeferredSlots<'a> { // this claimer its only writer until DONE (this fn's `# Safety`); // the caller reads it only after the join. let par = unsafe { std::slice::from_raw_parts_mut(c.par.0, c.par.1) }; - swiglu_into(&c.gate.data, &c.up.data, c.limit, par); + c.combine.write_par(par); if let Some(t_s) = t_s { self.swiglu_ns.fetch_add( t_s.elapsed().as_nanos() as u64, @@ -4294,6 +4381,23 @@ pub fn host_stack_runs( Ok(()) } +/// The form of a layer's routed experts — which stacks the layer holds and +/// the combine between them. An activation fact the architecture decides, +/// not a model: the host union serves either form of any layer's stacks, so +/// it lives here, beside the combines that read it. +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum ExpertForm { + /// Gated SwiGLU: `[gate, up, down]`, the combine `silu(gate·x) ⊙ (up·x)` + /// — `clamp(up, ±limit) · min(silu(gate), limit)` with a limit above + /// 1e-6; at or below it the plain combine ([`qdot::swiglu_clamp`]'s + /// contract). + Gated { limit: f32 }, + /// Ungated relu²: `[up, down]`, the combine `relu(up·x)²` + /// ([`relu2_into`]). A layer of this form holds no gate stack, and the + /// union reads its up where the gated form reads the gate and the up. + Relu2, +} + /// One stack of a union call, read off its [`ExpertStack`]: the weight type, /// the values a row contracts, its rows, the bytes of a row, the bytes of one /// input column in the type's encoding, and where its rows lie. @@ -4452,14 +4556,15 @@ impl UnionStack { } /// Where a union call's combines come from and go: slot `q` of expert `d` is -/// `silu(gate) · up` (clamped with `limit`) of its gate and up columns in -/// `gu`, quantized in `ty`'s encoding into slot `q` of `qc`. +/// its form's combine — `silu(gate) · up` (clamped with the limit) of its +/// gate and up columns, or `relu(up)²` of its up column — in `gu`, quantized +/// in `ty`'s encoding into slot `q` of `qc`. #[derive(Clone, Copy)] struct Combines<'c> { plan: UnionPlanView<'c>, gu: &'c [f32], ff: usize, - limit: Option, + form: ExpertForm, ty: GgmlType, qc: SharedQuantCols, } @@ -4475,15 +4580,24 @@ impl Combines<'_> { unsafe fn slot(&self, d: usize, q: usize) { let (o, m) = (self.plan.off[d], self.plan.off[d + 1] - self.plan.off[d]); let t = q - o; - let gate = &self.gu[(2 * o + t) * self.ff..][..self.ff]; - let up = &self.gu[(2 * o + m + t) * self.ff..][..self.ff]; PAR_BUF.with(|b| { let mut par = b.borrow_mut(); if par.len() < self.ff { par.resize(self.ff, 0.0); } let par = &mut par[..self.ff]; - swiglu_into(gate, up, self.limit, par); + match self.form { + ExpertForm::Gated { limit } => { + let gate = &self.gu[(2 * o + t) * self.ff..][..self.ff]; + let up = &self.gu[(2 * o + m + t) * self.ff..][..self.ff]; + swiglu_into(gate, up, Some(limit), par); + } + // A relu² layer's first pass wrote the up at the slot itself. + ExpertForm::Relu2 => { + let up = &self.gu[q * self.ff..][..self.ff]; + relu2_into(up, par); + } + } // SAFETY: the caller owns slot `q` of `qc`. unsafe { self.qc.quantize_into(self.ty, self.ff, par, q) }; }); @@ -4619,10 +4733,12 @@ enum UnionClaims<'c> { /// Which rows a union pass runs: every expert's gate and up (pair `2d` is /// expert `d`'s gate, `2d + 1` its up, over the columns of x that list it), -/// or every expert's down (pair `d`, over its combines). +/// every expert's up alone (pair `d`, a relu² layer's first pass), or every +/// expert's down (pair `d`, over its combines). #[derive(Clone, Copy, PartialEq, Eq)] enum UnionPass { GateUp, + Up, Down, } @@ -4657,6 +4773,7 @@ impl UnionRows<'_, '_> { fn split(&self, p: usize) -> (usize, usize) { match self.pass { UnionPass::GateUp => (p / 2, p % 2), + UnionPass::Up => (p, 1), UnionPass::Down => (p, 0), } } @@ -4680,7 +4797,7 @@ impl RowSet for UnionRows<'_, '_> { fn pairs(&self) -> usize { match self.pass { UnionPass::GateUp => 2 * self.plan.n(), - UnionPass::Down => self.plan.n(), + UnionPass::Up | UnionPass::Down => self.plan.n(), } } @@ -4752,6 +4869,7 @@ impl RowSet for UnionRows<'_, '_> { (self.inputs[s], self.x_cols, Some(self.plan.cols(d))), 2 * o + s * m, ), + UnionPass::Up => ((self.inputs[1], self.x_cols, Some(self.plan.cols(d))), o), UnionPass::Down => { // SAFETY: the method's contract — slots `o..o + m` are expert // `d`'s, inside the combines' slab, which holds every slot of @@ -4880,7 +4998,7 @@ pub(crate) struct UnionCall<'c, 'w> { plan: UnionPlanView<'c>, sources: [ExpertStack<'w>; 3], x: Tensor2View<'c>, - limit: Option, + form: ExpertForm, gate: UnionStack, up: UnionStack, down: UnionStack, @@ -4889,17 +5007,20 @@ pub(crate) struct UnionCall<'c, 'w> { } impl<'c, 'w> UnionCall<'c, 'w> { - /// The call over `plan`'s experts of `sources` (`[gate, up, down]`), for - /// `x`'s columns and experts `ff` wide: the stacks checked - /// ([`UnionStack::of_call`]) and every id of the plan one of every - /// stack's experts, each failure its named error before any pass runs. + /// The call over `plan`'s experts of `sources` (`[gate, up, down]` — for + /// a relu² layer the up again in the gate's slot, the form reading one + /// stack where the gated reads two), for `x`'s columns and experts as + /// wide as the up's rows: the stacks checked ([`UnionStack::of_call`]) + /// and every id of the plan one of every stack's experts, each failure + /// its named error before any pass runs. The layer's own width, never + /// the scratch's: a scratch may be made for a wider layer. pub(crate) fn new( plan: UnionPlanView<'c>, sources: [ExpertStack<'w>; 3], x: Tensor2View<'c>, - limit: Option, - ff: usize, + form: ExpertForm, ) -> Result { + let ff = sources[1].n; let [gate, up, down] = UnionStack::of_call(&sources, x.ne0(), ff)?; // The ids ascend: the last is the largest. if let Some(&e) = plan.ids.last() @@ -4914,7 +5035,7 @@ impl<'c, 'w> UnionCall<'c, 'w> { plan, sources, x, - limit, + form, gate, up, down, @@ -4928,6 +5049,16 @@ impl<'c, 'w> UnionCall<'c, 'w> { UnionCall { spread, ..self } } + /// The cells the gate/up pass writes: two columns a slot gated — the + /// gate's and the up's — one a slot for a relu² layer, the up's alone. + fn gu_cells(&self) -> usize { + let cells = self.plan.slots() * self.gate.n; + match self.form { + ExpertForm::Gated { .. } => 2 * cells, + ExpertForm::Relu2 => cells, + } + } + /// The call's passes, in order, through `slabs` into `out`: column `j` /// of `out` (`out[j·embd..(j + 1)·embd]`) is `lists[j]`'s weighted sum of /// its experts' downs of column `j` of x, from zero in list order. @@ -4936,8 +5067,9 @@ impl<'c, 'w> UnionCall<'c, 'w> { /// A call past [`DEFER_MAX_COLS`] columns runs five passes over the pool: /// x quantized — once more in the up's encoding when the up's rows read /// other bytes than the gate's ([`UnionStack::shares_input`]) — every - /// expert's gate and up rows, every slot's combine quantized, every - /// expert's down rows, every column's sum. A narrower one claims the + /// expert's gate and up rows (the up alone, one dispatch, for a relu² + /// layer), every slot's combine quantized, every expert's down rows, + /// every column's sum. A narrower one claims the /// quantizations inside its two row passes (x column by column, each /// expert's combines at once) and sums on the caller; with the deferral /// lever off it runs the quantize and combine passes too, four. Each @@ -4950,7 +5082,7 @@ impl<'c, 'w> UnionCall<'c, 'w> { out: &mut [f32], passes: &mut u64, ) -> Result<(), crate::ModelError> { - let (cols, ff, embd, slots) = (self.x.ne1(), self.gate.n, self.down.n, self.plan.slots()); + let (cols, embd, slots) = (self.x.ne1(), self.down.n, self.plan.slots()); assert_eq!(lists.len(), cols, "host union: one list a column of x"); assert_eq!( out.len(), @@ -4967,7 +5099,7 @@ impl<'c, 'w> UnionCall<'c, 'w> { "x in the up's encoding", )) }; - let gu = room(slabs.gu, 2 * slots * ff, "the gate/up slab"); + let gu = room(slabs.gu, self.gu_cells(), "the gate/up slab"); let qc = room(slabs.qc, slots * self.down.cb, "the combines' slab"); let store = room(slabs.store, slots * embd, "the down store"); // Poisoned like a dispatch's own buffers, so an unwritten cell shows. @@ -5077,8 +5209,8 @@ impl<'c, 'w> UnionCall<'c, 'w> { claims: Option>, passes: &mut u64, ) -> Result<(), crate::ModelError> { - let (ff, embd, slots) = (self.gate.n, self.down.n, self.plan.slots()); - let gu = room(gu, 2 * slots * ff, "the gate/up slab"); + let (embd, slots) = (self.down.n, self.plan.slots()); + let gu = room(gu, self.gu_cells(), "the gate/up slab"); let qc = room(qc, slots * self.down.cb, "the combines' slab"); let store = room(store, slots * embd, "the down store"); if poison() { @@ -5160,9 +5292,10 @@ impl<'c, 'w> UnionCall<'c, 'w> { /// Every expert's gate and up rows over the columns that list it, into /// `gu`, one dispatch; the up reads `xq_up` when given, else the gate's - /// `xq`. With `claim` x is quantized as the dispatch's first work, column - /// by column; without, `xq` and `xq_up` hold it. Returns the longest - /// claim pass at level >= 1. + /// `xq`. A relu² layer's pass reads the up alone: one pair an expert, + /// one column a slot. With `claim` x is quantized as the dispatch's first + /// work, column by column; without, `xq` and `xq_up` hold it. Returns the + /// longest claim pass at level >= 1. fn gate_up( &self, xq: &mut [u8], @@ -5189,6 +5322,14 @@ impl<'c, 'w> UnionCall<'c, 'w> { done: std::sync::atomic::AtomicUsize::new(0), claim_ns: std::sync::atomic::AtomicU64::new(0), }; + // A relu² layer's first pass reads the up alone: one pair an expert, + // the up's slots the store's shape. + let (pass, site, rec) = match self.form { + ExpertForm::Gated { .. } => { + (UnionPass::GateUp, "host_gate_up", &[self.gate, self.up][..]) + } + ExpertForm::Relu2 => (UnionPass::Up, "host_up", &[self.up][..]), + }; // SAFETY (construction site, referenced by every access): the slabs are // the caller's, borrowed for the whole dispatch, each holding the call's // columns or the plan's slots (`run`'s `room`). A pair writes only its @@ -5197,7 +5338,7 @@ impl<'c, 'w> UnionCall<'c, 'w> { // them — the quantize dispatch joined, or every claim counted done — // and each claim writes one column alone. let set = UnionRows { - pass: UnionPass::GateUp, + pass, plan: self.plan, stacks: [self.gate, self.up], sources: [self.sources[0], self.sources[1]], @@ -5218,8 +5359,8 @@ impl<'c, 'w> UnionCall<'c, 'w> { if let Some(t0) = t0 { let wall = t0.elapsed().as_nanos() as u64; record_union_rows( - "host_gate_up", - &[self.gate, self.up], + site, + rec, self.plan.n(), wall.saturating_sub(claim_ns), &pacc, @@ -5228,8 +5369,8 @@ impl<'c, 'w> UnionCall<'c, 'w> { Ok(claim_ns) } - /// The combines of the call: slot `q`'s `silu(gate) · up` of its columns - /// in `gu`, quantized in the down's encoding into slot `q` of `qc`. + /// The combines of the call: slot `q`'s form's combine of its columns in + /// `gu`, quantized in the down's encoding into slot `q` of `qc`. fn combines<'a>(&self, gu: &'a [f32], qc: &mut [u8]) -> Combines<'a> where 'c: 'a, @@ -5238,7 +5379,7 @@ impl<'c, 'w> UnionCall<'c, 'w> { plan: self.plan, gu, ff: self.gate.n, - limit: self.limit, + form: self.form, ty: self.down.ty, qc: SharedQuantCols::Bytes { cb: self.down.cb, @@ -6017,7 +6158,12 @@ mod tests { off: &[0, 1, 2], cols: &[0, 0], }; - let e = match UnionCall::new(plan, [gate, up, down], x.view(), None, ff) { + let e = match UnionCall::new( + plan, + [gate, up, down], + x.view(), + crate::ops::ExpertForm::Gated { limit: 0.0 }, + ) { Ok(_) => panic!("an id past the stacks must be refused"), Err(e) => e.to_string(), }; diff --git a/crates/model/tests/common/columns.rs b/crates/model/tests/common/columns.rs new file mode 100644 index 000000000..132644d56 --- /dev/null +++ b/crates/model/tests/common/columns.rs @@ -0,0 +1,28 @@ +//! The activation columns and the bit comparison the host-tier gates share. + +/// `n` seeded values in [-1, 1), every 61st scaled by 8 so a block's scale +/// is set by an outlier the way real activations set it. +pub fn seeded(n: usize, key: u64) -> Vec { + let mut s = key.wrapping_mul(0x9E37_79B9_7F4A_7C15) | 1; + (0..n) + .map(|i| { + s ^= s >> 12; + s ^= s << 25; + s ^= s >> 27; + let u = ((s.wrapping_mul(0x2545_F491_4F6C_DD1D) >> 40) as f32) / 8_388_608.0 - 1.0; + if i % 61 == 0 { u * 8.0 } else { u } + }) + .collect() +} + +/// Cells of `got` whose bits differ from `want`, a length mismatch counting +/// every cell. +pub fn diff_cells(got: &[f32], want: &[f32]) -> usize { + if got.len() != want.len() { + return got.len().max(want.len()); + } + got.iter() + .zip(want) + .filter(|(a, b)| a.to_bits() != b.to_bits()) + .count() +} diff --git a/crates/model/tests/common/panics.rs b/crates/model/tests/common/panics.rs new file mode 100644 index 000000000..44c004c98 --- /dev/null +++ b/crates/model/tests/common/panics.rs @@ -0,0 +1,14 @@ +//! A refusal that is a named panic, read back by the gates that hold it. + +/// The text of the panic `call` raises, `None` when it returns. A panic on a +/// pool worker arrives with its own text: the pool replays the first one. +pub fn panic_text(call: &mut dyn FnMut()) -> Option { + let payload = std::panic::catch_unwind(std::panic::AssertUnwindSafe(call)).err()?; + Some( + payload + .downcast_ref::() + .cloned() + .or_else(|| payload.downcast_ref::<&str>().map(|s| (*s).to_string())) + .unwrap_or_default(), + ) +} diff --git a/crates/model/tests/nemotron_h_moe_host.rs b/crates/model/tests/nemotron_h_moe_host.rs new file mode 100644 index 000000000..0a7535d00 --- /dev/null +++ b/crates/model/tests/nemotron_h_moe_host.rs @@ -0,0 +1,182 @@ +//! nemotron_h_moe host expert gate: what the host tier serves of the Bolt file +//! (`arch::nemotron_h_moe::host`), on the file's own stacks. +//! +//! - Every layer of the file through `host::layer`: a MoE layer builds as the +//! relu² form at its own expert count (the router's rows) and width (the +//! layer's `expert_feed_forward_length`), in the latent the header gives; a +//! Mamba-2 or attention layer is `None`; a layer past the model is refused +//! by name. +//! - The narrowest and the widest MoE layer of the file, through one union +//! scratch made for the widest (the widths differ from layer to layer), at +//! one column and at nine: nine distinct experts a column (Bolt's top 9), +//! every cell finite and the union call equal to `experts_into` of each +//! column, bit for bit, under both deferral arms. +//! +//! The kernels' own values are pinned elsewhere (`gate-qdot` for IQ2_XXS, the +//! union gate's relu² arm for the form against a transcription on qdot's +//! calls); this gate holds the wiring: the names, the latent, the per-layer +//! shapes and the capacity. +//! +//! `hw_`: needs the 24 GB file on the box at [`FILE`] (`just +//! gate-nemotron-h-moe-host`). Reads the stacks of the experts the columns +//! list, in two layers: seconds. + +#[path = "common/columns.rs"] +mod columns; + +use columns::{diff_cells, seeded}; +use gguf::Split; +use model::arch::nemotron_h_moe::host; +use model::arch::nemotron_h_moe::hparams::{Experts, Hparams, Kind}; +use model::moe::{HostLayer, HostScratch, UnionScratch}; +use model::ops::{self, ExpertForm, Tensor2}; +use model::r8file::R8Source; +use refset::arch::nemotron_h_moe::MODEL as FILE; + +/// Bolt's routed list: nine experts a token. +const USED: usize = 9; + +/// The column counts: one, and Bolt's nine. +const KS: [usize; 2] = [1, 9]; + +/// Column `j`'s list: [`USED`] distinct experts of the layer's `n_expert` +/// (every count of the file is coprime to 11, so the stride visits distinct +/// ones), weights in [0.05, 0.3). +fn list(j: usize, n_expert: usize) -> Vec<(u32, f32)> { + (0..USED) + .map(|i| { + let e = ((j * 37 + i * 11) % n_expert) as u32; + (e, 0.05 + 0.025 * ((i + j) % 10) as f32) + }) + .collect() +} + +#[test] +#[ignore = "hw: needs the Bolt 24GB file on the box (just gate-nemotron-h-moe-host)"] +fn hw_nemotron_h_moe_host() { + let split = Split::open(FILE).unwrap_or_else(|e| panic!("open {FILE}: {e}")); + let hp = Hparams::read(&split).unwrap_or_else(|e| panic!("hparams: {e}")); + let src = R8Source::rows(&split); + let mut failed: Vec = Vec::new(); + let mut check = |what: String, ok: bool| { + println!("{what}: {}", if ok { "PASS" } else { "FAIL" }); + if !ok { + failed.push(what); + } + }; + + // Every layer through the arch's one wiring. + let mut layers: Vec<(usize, Experts, HostLayer)> = Vec::new(); + let mut hosted_none = 0usize; + for l in 0..hp.n_layer { + let built = host::layer(src, &hp, l) + .unwrap_or_else(|e| panic!("layer {l} ({:?}): {e}", hp.kinds[l])); + match (hp.kinds[l], hp.experts[l], built) { + (Kind::Moe, Some(e), Some(layer)) => { + check( + format!( + "layer {l} builds as relu² at {} experts of {}", + e.n_expert, e.ff + ), + layer.form() == ExpertForm::Relu2 && layer.n_expert() == e.n_expert, + ); + layers.push((l, e, layer)); + } + (Kind::Mamba | Kind::Attention, None, None) => hosted_none += 1, + (kind, _, built) => check( + format!("layer {l} ({kind:?}) is hosted as {}", built.is_some()), + false, + ), + } + } + check( + format!( + "{} MoE layers built, {hosted_none} others hosted as none, of {} layers", + layers.len(), + hp.n_layer + ), + layers.len() + hosted_none == hp.n_layer && !layers.is_empty(), + ); + let past = host::layer(src, &hp, hp.n_layer) + .err() + .map(|e| e.to_string()) + .unwrap_or_default(); + check( + format!("a layer past the model is refused by name: {past:?}"), + past.contains("a layer inside the model's"), + ); + + // The narrowest and the widest layer through one scratch for the widest. + let by_width = |pick: fn(Experts, Experts) -> bool| { + layers + .iter() + .skip(1) + .fold( + &layers[0], + |best, cur| { + if pick(cur.1, best.1) { cur } else { best } + }, + ) + .0 + }; + let narrow = by_width(|a, b| a.ff < b.ff); + let wide = by_width(|a, b| a.ff > b.ff); + let ff_cap = layers.iter().map(|(_, e, _)| e.ff).max().unwrap(); + let mut us = UnionScratch::new_routed(hp.latent, ff_cap, USED, USED) + .expect("a scratch for the widest layer"); + for l in [narrow, wide] { + let (_, e, layer) = layers.iter().find(|(i, _, _)| *i == l).unwrap(); + let mut host = HostScratch::new(hp.latent, e.ff, USED).expect("a scratch of nine experts"); + for k in KS { + let x = Tensor2::from_vec( + hp.latent, + k, + (0..k) + .flat_map(|j| seeded(hp.latent, 0xb017 + (l * 16 + j) as u64)) + .collect(), + ); + let lists: Vec> = (0..k).map(|j| list(j, e.n_expert)).collect(); + let slices: Vec<&[(u32, f32)]> = lists.iter().map(Vec::as_slice).collect(); + let mut want = vec![f32::NAN; hp.latent * k]; + for j in 0..k { + let xj = Tensor2::from_vec(hp.latent, 1, x.col(j).to_vec()); + layer + .experts_into( + src, + &xj, + &lists[j], + &mut want[j * hp.latent..(j + 1) * hp.latent], + &mut host, + ) + .unwrap_or_else(|err| panic!("layer {l} experts_into column {j}: {err}")); + } + check( + format!("layer {l} k={k}: experts_into gives finite values"), + want.iter().all(|v| v.is_finite()), + ); + for (arm, on) in [("defer", true), ("prepass", false)] { + ops::set_defer_quant(Some(on)); + let mut got = vec![f32::NAN; hp.latent * k]; + let r = layer.experts_union_into(src, &x, &slices, &mut got, &mut us); + ops::set_defer_quant(None); + r.unwrap_or_else(|err| panic!("layer {l} k {k} {arm}: {err}")); + let d = diff_cells(&got, &want); + check( + format!( + "layer {l} (ff {}, {} experts) k={k} {arm}: union equals experts_into, \ + diff_cells={d}", + e.ff, e.n_expert + ), + d == 0, + ); + } + } + } + assert!(failed.is_empty(), "failed: {failed:#?}"); + println!( + "PASSED: nemotron_h_moe host — {} MoE layers build as relu², the rest as none; layers {narrow} \ + and {wide} (the narrowest and widest ff) through one scratch for ff {ff_cap}: union equal to \ + experts_into at k = 1, 9 under both deferral arms", + layers.len() + ); +} diff --git a/crates/model/tests/ops.rs b/crates/model/tests/ops.rs index 9c52042e2..305848e09 100644 --- a/crates/model/tests/ops.rs +++ b/crates/model/tests/ops.rs @@ -10,9 +10,12 @@ mod manifest; mod model_path; #[path = "common/oracle.rs"] mod oracle; +#[path = "common/panics.rs"] +mod panics; use gguf::GgmlType; use model::ops::{Tensor2, f32_tensor, matmul_q, rms_norm}; +use panics::panic_text; /// The slot-count observable and the deferral override are process-global, /// and cargo runs this file's tests in parallel — the tests that touch @@ -721,7 +724,10 @@ fn hw_group_swiglu_matches_caller_composition() { /// pars bit for bit — on both deferral arms, at one column and at six. A /// clamped combine (`GroupInput::SwigluClamp`) through either entry equals /// the caller composition over `qdot::swiglu_clamp` bit for bit, with the -/// limit set where it bites on both sides of `up`. +/// limit set where it bites on both sides of `up`; a relu² combine +/// (`GroupInput::Relu2`) equals the caller composition over `relu(up)²`, through the wide entry +/// (`matmul_q_group_cols_into`, ten columns) as well, and refuses a NaN or an infinity in an up +/// block by name through every entry. #[test] #[ignore = "hw: needs the box, the model file and $BLOOMERY_DATA/ref"] fn hw_group_into_matches_allocating_entries() { @@ -735,6 +741,20 @@ fn hw_group_into_matches_allocating_entries() { let (xs, xinf) = o.load("ffn_norm-1", 0); let xall = Tensor2::from_vec(xinf.ne[0] as usize, xinf.ne[1] as usize, xs); let x1 = Tensor2::from_vec(xall.ne0, 1, xall.col(0).to_vec()); + // Ten columns, past the widest claim: the group entry that takes a wide input + // (`matmul_q_group_cols_into`) produces each combine in the pre-pass. + let x10 = Tensor2::from_vec( + xall.ne0, + 10, + (0..10) + .flat_map(|j| { + xall.col(j % xall.ne1) + .iter() + .map(|v| v * (1.0 + j as f32 / 16.0)) + .collect::>() + }) + .collect(), + ); let names = [ "blk.1.ffn_gate_exps.weight", @@ -766,7 +786,9 @@ fn hw_group_into_matches_allocating_entries() { .collect() }; let (mut hits_hi, mut hits_lo, mut hits_silu) = (0usize, 0usize, 0usize); - for (x, label) in [(&x1, "ne1=1"), (&xall, "ne1=6")] { + let mut wide_relu2 = 0usize; + let mut refused = 0usize; + for (x, label) in [(&x1, "ne1=1"), (&xall, "ne1=6"), (&x10, "ne1=10")] { for (mode, arm) in [(Some(true), "deferred"), (Some(false), "pre-pass")] { model::ops::set_defer_quant(mode); let gu_views: Vec = experts @@ -861,15 +883,134 @@ fn hw_group_into_matches_allocating_entries() { ); } } + + // The relu² combine (the ungated experts' form, which reads the up + // alone): relu(up)² of each up block through either entry equals + // the caller composition, and its downs the plain group's over + // those pars. + let relu2_pars: Vec = (0..n) + .map(|i| { + let up = &gu[2 * i + 1]; + let data = up + .data + .iter() + .map(|&u| { + let r = u.max(0.0); + r * r + }) + .collect(); + Tensor2::from_vec(up.ne0, up.ne1, data) + }) + .collect(); + let relu2_refs: Vec<&Tensor2> = relu2_pars.iter().collect(); + let relu2_downs = matmul_q_group(&g, &d_refs, &relu2_refs).unwrap(); + let relu2: Vec = + (0..n).map(|i| GroupInput::Relu2(&gu[2 * i + 1])).collect(); + let (r_downs, r_pars) = matmul_q_group_swiglu(&g, &d_refs, &relu2).unwrap(); + let (mut rd_into, mut rp_into) = (blank(&downs), blank(&pars)); + matmul_q_group_swiglu_into("test", &d_ws, &relu2, &mut rd_into, &mut rp_into).unwrap(); + for i in 0..n { + for (got, what) in [(&r_pars[i], "allocating"), (&rp_into[i], "_into")] { + assert_eq!( + differ(&relu2_pars[i].data, &got.data), + 0, + "relu² par {i} ({label}, {arm}, {what}): must equal relu(up)²" + ); + } + for (got, what) in [(&r_downs[i], "allocating"), (&rd_into[i], "_into")] { + assert_eq!( + differ(&relu2_downs[i].data, &got.data), + 0, + "relu² down {i} ({label}, {arm}, {what}): must equal the caller composition" + ); + } + } + // A wide input's combines are produced column by column in the pre-pass. + if x.ne1 > model::ops::DEFER_MAX_COLS { + let (mut cd, mut cp) = (blank(&downs), blank(&pars)); + model::ops::matmul_q_group_cols_into("test", &d_ws, &relu2, &mut cd, &mut cp) + .unwrap(); + for i in 0..n { + assert_eq!( + differ(&relu2_pars[i].data, &cp[i].data), + 0, + "relu² par {i} ({label}, {arm}, wide): must equal relu(up)²" + ); + assert_eq!( + differ(&relu2_downs[i].data, &cd[i].data), + 0, + "relu² down {i} ({label}, {arm}, wide): must equal the caller composition" + ); + } + wide_relu2 += 1; + } + // A NaN or an infinity in an up block is refused by name, never turned into zero + // by the relu: the last cell of the first expert's up, through every entry. + for (bad, what) in [ + (f32::NAN, "NaN"), + (f32::INFINITY, "+inf"), + (f32::NEG_INFINITY, "-inf"), + ] { + let mut ups: Vec = (0..n) + .map(|i| { + Tensor2::from_vec( + gu[2 * i + 1].ne0, + gu[2 * i + 1].ne1, + gu[2 * i + 1].data.clone(), + ) + }) + .collect(); + let last = ups[0].data.len() - 1; + ups[0].data[last] = bad; + let inputs: Vec = ups.iter().map(GroupInput::Relu2).collect(); + // The wide entry takes an input past the widest claim. + let entries: &[&str] = if x.ne1 > model::ops::DEFER_MAX_COLS { + &["allocating", "_into", "wide"] + } else { + &["allocating", "_into"] + }; + for &entry in entries { + let mut call = || { + let (mut od, mut op) = (blank(&downs), blank(&pars)); + let _ = match entry { + "allocating" => matmul_q_group_swiglu(&g, &d_refs, &inputs).map(drop), + "_into" => { + matmul_q_group_swiglu_into("test", &d_ws, &inputs, &mut od, &mut op) + .map(drop) + } + _ => model::ops::matmul_q_group_cols_into( + "test", &d_ws, &inputs, &mut od, &mut op, + ) + .map(drop), + }; + }; + let text = panic_text(&mut call); + assert!( + text.as_deref() + .is_some_and(|t| t.contains("relu2: non-finite activation value")), + "relu² with {what} in an up block ({label}, {arm}, {entry}) must panic naming \ + the non-finite value, got {text:?}" + ); + refused += 1; + } + } } } model::ops::set_defer_quant(None); + assert!( + refused == 3 * 2 * (2 + 2 + 3), + "every non-finite refusal clause must run: three values, two arms, two entries at ne1 1 and 6, three at 10 (got {refused})" + ); + assert!( + wide_relu2 > 0, + "the wide entry's relu² clause must run (a column count past DEFER_MAX_COLS)" + ); assert!( hits_hi > 0 && hits_lo > 0 && hits_silu > 0, "the limit must bite on every side (up > L {hits_hi}, up < -L {hits_lo}, silu > L {hits_silu})" ); eprintln!( - "_into == allocating entries, bit for bit: {n} experts' gate/up, down and pars, plain and clamped \ - (up > L {hits_hi}, up < -L {hits_lo}, silu > L {hits_silu}), ne1 1 and 6, both deferral arms" + "_into == allocating entries, bit for bit: {n} experts' gate/up, down and pars, plain, clamped \ + (up > L {hits_hi}, up < -L {hits_lo}, silu > L {hits_silu}) and relu², ne1 1, 6 and 10 (the wide entry's pre-pass at 10), both deferral arms" ); } diff --git a/crates/model/tests/union.rs b/crates/model/tests/union.rs index 9a379e83c..15ee3c700 100644 --- a/crates/model/tests/union.rs +++ b/crates/model/tests/union.rs @@ -41,6 +41,15 @@ //! combine in one pass and every down in one row dispatch, lanes cut by the //! tile's cost; the k <= 8 cases above claim their quantizations instead. //! +//! The relu² form — Bolt's ungated experts, an arm of the same union — is held on two +//! synthetic IQ2_XXS layers of different widths (no model file), served through one scratch +//! made for the wider: at k = 1, 8, 9 and 64 under both deferral arms the union call, +//! `experts_into` of every column and a transcription on qdot's own calls (`x` quantized, +//! each up row dotted, `relu(u)²`, the combine quantized, each down row dotted, the +//! list-order sum) are the same bits, in the dispatches the call is owed; and the scratch's +//! expert width is a capacity, so a scratch narrower than the layer, of another embd, or +//! with no room for a combine column is refused by name. +//! //! The free entry `moe::experts_union_into` (one file, a `MoeBlockPlan`) is //! held to its own `experts_into` on V2-Lite's block 1 at k = 6, over seeded //! activation columns with lists overlapping by construction. Then every @@ -98,6 +107,10 @@ //! `hw_`: needs the V4.1 shards, the oracle set and V2-Lite on the box //! (`just gate-union`). +#[path = "common/columns.rs"] +mod columns; +#[path = "common/panics.rs"] +mod panics; #[path = "common/r8layer.rs"] mod r8layer; #[path = "common/v41set.rs"] @@ -105,16 +118,18 @@ mod v41set; use std::sync::Arc; +use columns::{diff_cells, seeded}; use gguf::{GgmlType, Split, Weights}; use model::arch::deepseek41::host; use model::arch::deepseek41::hparams::Hparams; use model::moe::{ - HostLayer, HostLayerSpec, HostScratch, UNION_MAX_COLS, UNION_TAIL_MAX_GROUPS, UnionScratch, - union_batch_slots, + HostLayer, HostLayerSpec, HostScratch, Relu2Spec, UNION_MAX_COLS, UNION_TAIL_MAX_GROUPS, + UnionScratch, union_batch_slots, }; use model::ops::{self, RowLayout, Tensor2, Tensor2View}; use model::placement::workstation; use model::r8file::{HostR8, R8Source, Sidecar}; +use panics::panic_text; /// The oracle set: the V4.1 node dumps' 5-token batch set. const SET: &str = refset::arch::deepseek41::BATCH; @@ -220,18 +235,6 @@ fn dispatch_note(scratch: u64, pool: Option, want: u64) -> String { format!("dispatches={scratch} pool={pool} want={want}") } -/// Cells of `got` whose bits differ from `want`, a length mismatch counting -/// every cell. -fn diff_cells(got: &[f32], want: &[f32]) -> usize { - if got.len() != want.len() { - return got.len().max(want.len()); - } - got.iter() - .zip(want) - .filter(|(a, b)| a.to_bits() != b.to_bits()) - .count() -} - /// The V4.1 cases of one layer: the set's first k tokens for k <= 5, and /// past them the columns of the module doc. fn v41_cases(x_all: &[f32], lists: &[Vec<(u32, f32)>], embd: usize) -> Vec { @@ -1056,21 +1059,6 @@ fn v2lite_path() -> String { .unwrap_or_else(|_| "/models/small/DeepSeek-V2-Lite-Chat.Q3_K_M.gguf".into()) } -/// `n` seeded values in [-1, 1), every 61st scaled by 8 so a block's scale -/// is set by an outlier the way real activations set it. -fn seeded(n: usize, key: u64) -> Vec { - let mut s = key.wrapping_mul(0x9E37_79B9_7F4A_7C15) | 1; - (0..n) - .map(|i| { - s ^= s >> 12; - s ^= s << 25; - s ^= s >> 27; - let u = ((s.wrapping_mul(0x2545_F491_4F6C_DD1D) >> 40) as f32) / 8_388_608.0 - 1.0; - if i % 61 == 0 { u * 8.0 } else { u } - }) - .collect() -} - #[test] #[ignore = "hw: needs the box and the V2-Lite model file"] fn hw_union_file_matches_per_column_v2lite() { @@ -1266,33 +1254,22 @@ const Q8_DOWN: &str = "blk.0.ffn_down_exps.weight"; /// f32's unit roundoff. const U: f64 = 1.0 / 16_777_216.0; -/// One routed layer — Q4_K gate and up, a Q8_0 or Q6_K down — written with `gguf::write` into -/// its own directory (removed on drop), and the stacks' bytes the file holds. -struct Q8Layer { +/// A one-file gguf of synthetic stacks, `(name, type, [k, n, n_expert], bytes)` each, written +/// with `gguf::write` into a directory of its own (removed on drop). +struct StackFile { dir: std::path::PathBuf, source: std::path::PathBuf, - gate: Vec, - up: Vec, - down: Vec, } -impl Q8Layer { - fn write(embd: usize, ff: usize, n_expert: usize, down_ty: GgmlType) -> Q8Layer { +impl StackFile { + fn write(tag: &str, stacks: &[(&str, GgmlType, [usize; 3], &[u8])]) -> StackFile { use gguf::write::{Layout, TensorDecl, Writer}; - let dir = std::env::temp_dir().join(format!("q8layer-{down_ty}-{}", std::process::id())); + let dir = std::env::temp_dir().join(format!("{tag}-{}", std::process::id())); if dir.exists() { std::fs::remove_dir_all(&dir).unwrap(); } std::fs::create_dir_all(&dir).unwrap(); let source = dir.join("layer.gguf"); - let gate = q8_stack(GgmlType::Q4_K, [embd, ff, n_expert], 0x6a7e); - let up = q8_stack(GgmlType::Q4_K, [embd, ff, n_expert], 0x0b); - let down = q8_stack(down_ty, [ff, embd, n_expert], 0xd0); - let stacks = [ - (Q8_GATE, GgmlType::Q4_K, [embd, ff, n_expert], &gate), - (Q8_UP, GgmlType::Q4_K, [embd, ff, n_expert], &up), - (Q8_DOWN, down_ty, [ff, embd, n_expert], &down), - ]; let decls: Vec = stacks .iter() .map(|&(name, ty, dims, b)| TensorDecl { @@ -1305,29 +1282,54 @@ impl Q8Layer { let layout = Layout::new(&[], decls).unwrap(); let file = std::io::BufWriter::new(std::fs::File::create(&source).unwrap()); let mut w = Writer::new(file, layout).unwrap(); - for &(name, _, _, b) in &stacks { + for &(name, _, _, b) in stacks { w.tensor(name, b).unwrap(); } w.finish().unwrap(); - Q8Layer { - dir, - source, - gate, - up, - down, - } + StackFile { dir, source } } } -impl Drop for Q8Layer { +impl Drop for StackFile { fn drop(&mut self) { let _ = std::fs::remove_dir_all(&self.dir); } } -/// A stack `[k, n, n_expert]` of `ty` (Q4_K, Q6_K or Q8_0): xorshift bytes, each block's scales -/// set finite — Q4_K's `d` 2^-7 and `dmin` 2^-8, Q6_K's `d` 2^-10 (its sixteen i8 scales the -/// xorshift's), Q8_0's `d` of either sign at 2^-7 .. 2^-3. +/// One routed layer — Q4_K gate and up, a Q8_0 or Q6_K down — and the stacks' bytes its file holds. +struct Q8Layer { + file: StackFile, + gate: Vec, + up: Vec, + down: Vec, +} + +impl Q8Layer { + fn write(embd: usize, ff: usize, n_expert: usize, down_ty: GgmlType) -> Q8Layer { + let gate = q8_stack(GgmlType::Q4_K, [embd, ff, n_expert], 0x6a7e); + let up = q8_stack(GgmlType::Q4_K, [embd, ff, n_expert], 0x0b); + let down = q8_stack(down_ty, [ff, embd, n_expert], 0xd0); + let file = StackFile::write( + &format!("q8layer-{down_ty}"), + &[ + (Q8_GATE, GgmlType::Q4_K, [embd, ff, n_expert], &gate), + (Q8_UP, GgmlType::Q4_K, [embd, ff, n_expert], &up), + (Q8_DOWN, down_ty, [ff, embd, n_expert], &down), + ], + ); + Q8Layer { + file, + gate, + up, + down, + } + } +} + +/// A stack `[k, n, n_expert]` of `ty` (Q4_K, Q6_K, Q8_0 or IQ2_XXS): xorshift bytes, each +/// block's scales set finite — Q4_K's `d` 2^-7 and `dmin` 2^-8, Q6_K's `d` 2^-10 (its sixteen i8 +/// scales the xorshift's), Q8_0's `d` of either sign at 2^-7 .. 2^-3, IQ2_XXS's `d` 2^-10 (every +/// other bit of its block is a valid grid index, sign or scale). fn q8_stack(ty: GgmlType, [k, n, n_expert]: [usize; 3], seed: u64) -> Vec { let (bs, ts) = ( ty.blck_size().unwrap() as usize, @@ -1350,6 +1352,7 @@ fn q8_stack(ty: GgmlType, [k, n, n_expert]: [usize; 3], seed: u64) -> Vec { blk[2..4].copy_from_slice(&0x1C00u16.to_le_bytes()); } GgmlType::Q6_K => blk[208..210].copy_from_slice(&0x1400u16.to_le_bytes()), + GgmlType::IQ2_XXS => blk[0..2].copy_from_slice(&0x1400u16.to_le_bytes()), GgmlType::Q8_0 => { let r = next(); let (exp, mant, sign) = ( @@ -1393,7 +1396,7 @@ fn gamma(n: f64) -> f64 { fn hw_union_q8_0_down_matches_dequant() { let (embd, ff, n_expert) = Q8_LAYER; let layer = Q8Layer::write(embd, ff, n_expert, GgmlType::Q8_0); - let split = Split::open(&layer.source).unwrap(); + let split = Split::open(&layer.file.source).unwrap(); let src = R8Source::rows(&split); let spec = HostLayerSpec { gate: Q8_GATE, @@ -1545,7 +1548,7 @@ fn q6_lists() -> Vec> { fn hw_union_q6_k_down_matches_per_column() { let (embd, ff, n_expert) = Q6_LAYER; let layer = Q8Layer::write(embd, ff, n_expert, GgmlType::Q6_K); - let split = Split::open(&layer.source).unwrap(); + let split = Split::open(&layer.file.source).unwrap(); let src = R8Source::rows(&split); let spec = HostLayerSpec { gate: Q8_GATE, @@ -1630,3 +1633,315 @@ fn hw_union_q6_k_down_matches_per_column() { and the lane packs' prefetch" ); } + +/// Bolt's routed layers in the one shape that differs from layer to layer: the latent the +/// experts run in, and two layers' own (ff, expert count) — the scratch is made for the wider. +const RELU2_EMBD: usize = 1024; +const RELU2_LAYERS: [(usize, usize); 2] = [(1536, 12), (2560, 16)]; + +/// Bolt's routed list: nine experts a token, one past [`LIST`]. +const RELU2_USED: usize = 9; + +/// The relu² cases' column counts: one column, the widest call that claims, the narrowest that +/// does not, and one that runs experts through several tile runs. +const RELU2_KS: [usize; 4] = [1, 8, 9, 64]; + +/// One relu² routed layer — IQ2_XXS up and down, no gate — and the stacks' bytes its file holds. +struct Relu2Layer { + file: StackFile, + up: Vec, + down: Vec, +} + +impl Relu2Layer { + fn write(embd: usize, ff: usize, n_expert: usize) -> Relu2Layer { + let ty = GgmlType::IQ2_XXS; + let up = q8_stack(ty, [embd, ff, n_expert], 0x2e1); + let down = q8_stack(ty, [ff, embd, n_expert], 0xd2e1); + let file = StackFile::write( + &format!("relu2layer-{ff}"), + &[ + (Q8_UP, ty, [embd, ff, n_expert], &up), + (Q8_DOWN, ty, [ff, embd, n_expert], &down), + ], + ); + Relu2Layer { file, up, down } + } +} + +/// The bytes of one IQ2_XXS row of `k` values. +fn iq2_row_bytes(k: usize) -> usize { + let ty = GgmlType::IQ2_XXS; + k / ty.blck_size().unwrap() as usize * ty.type_size().unwrap() as usize +} + +/// The up output of expert `e` on one column, before the relu² — what `HostScratch::up` holds: +/// `x` quantized, each of the expert's `ff` up rows dotted with it on qdot's own call. +fn relu2_up(layer: &Relu2Layer, x: &[f32], e: usize, ff: usize) -> Vec { + let ty = GgmlType::IQ2_XXS; + let embd = x.len(); + let row_up = iq2_row_bytes(embd); + let mut xq = vec![0u8; qdot::col_bytes(ty, embd)]; + qdot::quantize_col(ty, x, &mut xq); + (0..ff) + .map(|r| { + let at = (e * ff + r) * row_up; + qdot::dot_row(ty, &layer.up[at..at + row_up], &xq, embd).unwrap() + }) + .collect() +} + +/// One column of a relu² layer, transcribed from the contract on qdot's own calls and nothing of +/// the union's or the leg's: `x` quantized once, each listed expert's up row dotted with it, +/// `relu(u)²` of that row's f32, the combine quantized, each down row dotted with it, and the +/// list-order sum `o += w · d` from zero. +fn relu2_column(layer: &Relu2Layer, x: &[f32], list: &[(u32, f32)], ff: usize) -> Vec { + let ty = GgmlType::IQ2_XXS; + let embd = x.len(); + let row_down = iq2_row_bytes(ff); + let mut hq = vec![0u8; qdot::col_bytes(ty, ff)]; + let mut out = vec![0.0f32; embd]; + for &(e, w) in list { + let e = e as usize; + let h: Vec = relu2_up(layer, x, e, ff) + .into_iter() + .map(|u| { + let relu = u.max(0.0); + relu * relu + }) + .collect(); + qdot::quantize_col(ty, &h, &mut hq); + for (o, cell) in out.iter_mut().enumerate() { + let at = (e * embd + o) * row_down; + let d = qdot::dot_row(ty, &layer.down[at..at + row_down], &hq, ff).unwrap(); + *cell += w * d; + } + } + out +} + +/// The relu² form — Bolt's ungated experts — through the common union: on two layers of +/// different widths and expert counts (IQ2_XXS up and down, the latent 1024 wide), one scratch +/// made for the wider serves both, and at k = 1, 8, 9 and 64 under both deferral arms the union +/// call, `experts_into` of every column and `relu2_column`'s transcription on qdot's own calls +/// are the same bits, in the dispatches the call is owed, and `HostScratch::up` holds each expert's +/// up output of the last column's call in list order. A scratch exact for the narrower +/// layer gives the same bits as the capacity scratch. Then the named refusals of the widths: a +/// scratch narrower than the layer's ff, one of another embd, and one whose capacity holds no +/// column encoding. +#[test] +#[ignore = "hw: the box's CPU (qdot's fused kernels run on AVX2); reads no model file"] +fn hw_union_relu2_matches_transcription() { + let embd = RELU2_EMBD; + let ff_cap = RELU2_LAYERS.iter().map(|&(ff, _)| ff).max().unwrap(); + let mut us = UnionScratch::new_routed(embd, ff_cap, UNION_MAX_COLS, RELU2_USED) + .expect("a scratch for the wider layer"); + let mut failed: Vec<(usize, usize, &str)> = Vec::new(); + let mut miscounted: Vec<(usize, usize, &str)> = Vec::new(); + let mut last = None; + for (ff, n_expert) in RELU2_LAYERS { + let layer = Relu2Layer::write(embd, ff, n_expert); + let split = Split::open(&layer.file.source).unwrap(); + let src = R8Source::rows(&split); + let host_layer = HostLayer::build_relu2( + src, + &Relu2Spec { + up: Q8_UP, + down: Q8_DOWN, + n_expert, + embd, + ff, + }, + ) + .unwrap_or_else(|e| panic!("a relu² layer of IQ2_XXS must build: {e}")); + assert_eq!( + host_layer.form(), + ops::ExpertForm::Relu2, + "build_relu2 gives the relu² form" + ); + let x_all: Vec = (0..4).flat_map(|t| seeded(embd, 0x2e1 + t)).collect(); + let mut host = HostScratch::new(embd, ff, RELU2_USED).expect("a scratch of nine experts"); + let mut exact = UnionScratch::new_routed(embd, ff, UNION_MAX_COLS, RELU2_USED) + .expect("a scratch exact for the layer"); + for k in RELU2_KS { + let mut case = wide_case(&x_all, embd, k, n_expert, n_expert); + // One column carries Bolt's full nine. + case.lists[0] = (0..RELU2_USED as u32) + .map(|e| (e, 0.1 + 0.01 * e as f32)) + .collect(); + let want: Vec = (0..k) + .flat_map(|j| relu2_column(&layer, case.x.col(j), &case.lists[j], ff)) + .collect(); + assert!( + want.iter().all(|v| v.is_finite()), + "ff {ff} k {k}: the transcription gives finite values" + ); + let (lo, hi, past) = cols_per_expert(&case); + let column = per_column(&host_layer, src, &case, &mut host); + let dd = diff_cells(&column, &want); + if dd != 0 { + failed.push((ff, k, "experts_into")); + } + // The scratch's accessor holds each expert's up output of the last column's call, + // in list order: the contract the gated forms keep. + let last_col = k - 1; + let du: usize = case.lists[last_col] + .iter() + .enumerate() + .map(|(i, &(e, _))| { + let up = relu2_up(&layer, case.x.col(last_col), e as usize, ff); + diff_cells(host.up(i).col(0), &up) + }) + .sum(); + if du != 0 { + failed.push((ff, k, "scratch up")); + } + let lists = case.slices(); + let mut line = format!( + "relu2 ff={ff} k={k} slots={} union={} cols_per_expert={lo}..{hi} \ + past_one_run={past} experts_into: diff_cells={dd} scratch_up={du}", + case.slots(), + case.union() + ); + for (arm, on) in ARMS { + ops::set_defer_quant(Some(on)); + let mut got = vec![f32::NAN; embd * k]; + let (r, ds, dp) = dispatched(&mut us, |us| { + host_layer.experts_union_into(src, &case.x, &lists, &mut got, us) + }); + let mut got_exact = vec![f32::NAN; embd * k]; + let r_exact = + host_layer.experts_union_into(src, &case.x, &lists, &mut got_exact, &mut exact); + ops::set_defer_quant(None); + r.unwrap_or_else(|e| panic!("relu2 ff {ff} k {k} {arm}: {e}")); + r_exact.unwrap_or_else(|e| panic!("relu2 ff {ff} k {k} {arm} exact: {e}")); + let (d, de) = (diff_cells(&got, &want), diff_cells(&got_exact, &want)); + let want_d = call_dispatches(k, case.slots(), on); + line += &format!( + " {arm}: diff_cells={d} exact_scratch={de} {}", + dispatch_note(ds, dp, want_d) + ); + if d != 0 { + failed.push((ff, k, arm)); + } + if de != 0 { + failed.push((ff, k, "exact scratch")); + } + if !dispatches_ok(ds, dp, want_d) { + miscounted.push((ff, k, arm)); + } + } + println!("{line}"); + } + last = Some((layer, ff)); + } + assert!( + failed.is_empty(), + "{} (ff, k, call) relu² cases differ from the transcription: {failed:?}", + failed.len() + ); + assert!( + miscounted.is_empty(), + "{} (ff, k, arm) relu² calls issued other than their dispatches: {miscounted:?}", + miscounted.len() + ); + + // The refusals, on the wider layer (the last built). + let (layer, ff) = last.expect("two layers ran"); + let split = Split::open(&layer.file.source).unwrap(); + let src = R8Source::rows(&split); + let host_layer = HostLayer::build_relu2( + src, + &Relu2Spec { + up: Q8_UP, + down: Q8_DOWN, + n_expert: RELU2_LAYERS[1].1, + embd, + ff, + }, + ) + .unwrap(); + let one = [(0u32, 1.0f32)]; + let x = Tensor2::zeros(embd, 1); + let mut out = vec![0.0f32; embd]; + for (what, e, ff_made, want) in [ + ( + "scratch narrower than the layer's ff", + embd, + RELU2_LAYERS[0].0, + "scratch must be made for the block's widths", + ), + ( + "scratch whose capacity holds no column encoding", + embd, + ff + 1, + "scratch must be made for the block's widths", + ), + ( + "scratch of another embd", + embd / 2, + ff, + "the scratch must be made for the block's width", + ), + ] { + let mut other = UnionScratch::new_routed(e, ff_made, SMALL_COLS, RELU2_USED) + .unwrap_or_else(|err| panic!("{what}: the scratch itself must build: {err}")); + let got = refusal( + host_layer.experts_union_into(src, &x, &[&one[..]], &mut out, &mut other), + what, + ); + assert!( + got.contains(want), + "{what}: refusal must name {want:?}, got {got:?}" + ); + println!("relu2 refusal ({what}): {got}"); + } + // An up row that overflows to an infinity is refused by name by the combine, never turned + // into zero by the relu: columns of ±3e38 against IQ2_XXS rows overflow the f32 dot. The + // one-column leg, the claimed union call and the wide union call each refuse it. + let hot = |k: usize| { + Tensor2::from_vec( + embd, + k, + (0..k * embd) + .map(|i| { + if (i / 7) % 2 == 0 { + 3.0e38f32 + } else { + -3.0e38f32 + } + }) + .collect(), + ) + }; + let nine: Vec<(u32, f32)> = (0..RELU2_USED as u32).map(|e| (e, 0.1)).collect(); + let overflow = |what: &str, call: &mut dyn FnMut()| { + let text = panic_text(call); + assert!( + text.as_deref() + .is_some_and(|t| t.contains("relu2: non-finite activation value")), + "{what}: an overflowing up row must panic naming the non-finite value, got {text:?}" + ); + println!("relu2 refusal ({what}): {}", text.unwrap_or_default()); + }; + let mut host = HostScratch::new(embd, ff, RELU2_USED).unwrap(); + overflow("experts_into", &mut || { + let mut out = vec![0.0f32; embd]; + let _ = host_layer.experts_into(src, &hot(1), &nine, &mut out, &mut host); + }); + for k in [1, SMALL_COLS] { + let x = hot(k); + let lists: Vec<&[(u32, f32)]> = vec![&nine[..]; k]; + let mut us = UnionScratch::new_routed(embd, ff, SMALL_COLS, RELU2_USED).unwrap(); + overflow(&format!("experts_union_into k={k}"), &mut || { + let mut out = vec![0.0f32; embd * k]; + let _ = host_layer.experts_union_into(src, &x, &lists, &mut out, &mut us); + }); + } + println!( + "PASSED: union relu² — IQ2_XXS up and down, ungated: the union call at k = 1, 8, 9, 64 on \ + two layers of different widths through one scratch made for the wider equal to \ + experts_into and to the transcription on qdot's calls, bit for bit under both deferral \ + arms and in their dispatches; the widths' refusals by name, and an overflowing up row refused by name" + ); +} diff --git a/justfile b/justfile index a2b4ac5bb..21051a28f 100644 --- a/justfile +++ b/justfile @@ -2648,6 +2648,14 @@ gate-nemotron-meta: gate-nemotron-h-moe-meta: ./tools/box.sh 'bash tools/gate.sh --release -p bloomery-model --lib -- arch::nemotron_h_moe --nocapture && bash tools/gate.sh --release -p bloomery-model --test nemotron_h_moe_meta -- --ignored --nocapture' +# The Bolt file's routed experts on the host tier (arch::nemotron_h_moe::host, the relu² form of the common union): every +# layer of the 24GB GGUF at the box's path through `host::layer` (a MoE layer builds as relu² at its own expert count and +# width, a Mamba-2 or attention layer as none, a layer past the model refused by name), then the file's narrowest and widest +# MoE layer through one union scratch made for the widest, at one column and nine, equal to experts_into bit for bit under +# both deferral arms. Reads the stacks of the experts the columns list: seconds. +gate-nemotron-h-moe-host: + ./tools/box.sh 'bash tools/gate.sh --release -p bloomery-model --test nemotron_h_moe_host -- --ignored --nocapture' + # gemma-embedding2 (EmbeddingGemma 2) header gate: the reader's refusals on synthetic headers (a full # layer at the sliding head, a missing tensor, another pooling, causal attention, a key head count # that does not divide the heads, an odd rope, another tokenizer, a missing key) and the coverage diff --git a/tools/gate-paths.tsv b/tools/gate-paths.tsv index 031b75274..aa8f36e79 100644 --- a/tools/gate-paths.tsv +++ b/tools/gate-paths.tsv @@ -98,18 +98,18 @@ tools/census/stubs.tsv gate-gpu-routed-legs the census manifest (refset::census: # files a gate opens. One row: every gate that links the modules, less the gates whose bins only read tensors # through gguf::Gguf or build synthetic blocks and call no arch module (their own code, and the bloomery-gpu # kernel modules and bloomery-gpu-gates helpers they call, name no model::arch:: path). the fixture generator, arch/coverage.rs and arch/qwen35moe/mtp.rs are carved out to their own rows below: their readers are bounded (a leaf the family's fixture module names; the place/engine/plan modules that call them). -crates/model/src/arch/{mod.rs,dspark.rs,classify.rs,stems.rs,deepseek2/**,qwen3moe/**,glm5next/**} gate-gpu-p4 gate-gpu-p6 gate-gpu-kquant gate-gpu-iq-sel gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-ptx-spill gate-dspark-read gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-q38-serve-rules gate-gpu-block gate-gpu-nvtier gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p5 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-cand gate-gpu-iq gate-gpu-q6k-ids gate-gpu-xstream gate-gpu-mimo2-flash gate-gpu-ple gate-gpu-vision gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-minicpm-text gate-gpu-mamba2 gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower gate-gpu-gemma-embedding-head256 gate-gpu-qwen38-nodes-remote gate-qwen3-plan gate-gpu-gemma4a gate-fixture-budget gate-gpu-gemma-embedding-e2e gate-gpu-gemma-embedding-serve every gate that links an arch module, less the gguf-only kernel gates (above): a family's code runs wherever any family's file is opened +crates/model/src/arch/{mod.rs,dspark.rs,classify.rs,stems.rs,deepseek2/**,qwen3moe/**,glm5next/**} gate-gpu-p4 gate-gpu-p6 gate-gpu-kquant gate-gpu-iq-sel gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-ptx-spill gate-dspark-read gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-q38-serve-rules gate-gpu-block gate-gpu-nvtier gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p5 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-cand gate-gpu-iq gate-gpu-q6k-ids gate-gpu-xstream gate-gpu-mimo2-flash gate-gpu-ple gate-gpu-vision gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-minicpm-text gate-gpu-mamba2 gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower gate-gpu-gemma-embedding-head256 gate-gpu-qwen38-nodes-remote gate-qwen3-plan gate-gpu-gemma4a gate-fixture-budget gate-gpu-gemma-embedding-e2e gate-gpu-gemma-embedding-serve gate-nemotron-h-moe-host every gate that links an arch module, less the gguf-only kernel gates (above): a family's code runs wherever any family's file is opened crates/model/src/arch/mimo2/** gate-ops gate-mimo2-meta gate-gpu-mimo2-flash gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-ds41-bind gate-ds41-place gate-gpu-routed-legs gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-minicpm-text weekly-gpu-mimo2-e2e weekly-gpu-mimo2-serve gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-qwen38-nodes-remote gate-qwen3-plan gate-fixture-budget gate-gpu-gemma-embedding-serve gate-gpu-gemma-embedding-e2e MiMo-V2.6-Flash's reader and plan: its code runs only where a mimo2 file opens (its meta test, its flash gate) in bloomery-model's own unit tests (gate-ops runs --lib) and in bloomery-serve's mimo2 seat, which every serve gate's bin links except gate-gpu-p6 gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-clef-hidden gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-qwen3moe-e2e gate-fixture gate-gpu-head gate-gpu-lib gate-gpu-ds41-lib gate-q38-serve-rules gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency: they open no mimo2 file: only model::arch's dispatch by the file's general.architecture (arch/mod.rs) reaches it crates/model/src/arch/nemotron_h/** gate-ops gate-nemotron-meta gate-fixture gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-iq-sel gate-placement gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-ds41-bind gate-ds41-place gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-qwen38-nodes-remote gate-qwen3-plan gate-fixture-budget gate-gpu-gemma-embedding-e2e gate-gpu-gemma-embedding-serve Nemotron 3 Nano 4B's header reader and fixture spec: its code runs where a nemotron_h file is described (its meta test), where its fixture is planned and verified (the fixture generator's gate), in bloomery-model's own unit tests, where the card body reads the layer kinds and widths at load (the end-to-end gate), and behind the arch dispatch and, through its plan (place.rs), in bloomery-serve's nemotron seat, which gate-gpu-nemotron-serve starts and every serve gate's bin links except gate-gpu-e2e gate-gpu-hybrid gate-gpu-p6 gate-gpu-ds41-moe gate-gpu-moe gate-gpu-p8 gate-gpu-block gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen35moe-moe gate-gpu-qwen3moe-router gate-qwen35moe-meta gate-gpu-qwen35-dspark gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-minicpm-text gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-types gate-gpu-head gate-gpu-routed-legs gate-gpu-ds41-lib gate-gpu-lib gate-q38-serve-rules gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-nvtier gate-gpu-ds41-residency-a gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-long gate-gpu-ds41-chat gate-gpu-ds41-media gate-gpu-ds41-prefill gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-ds41-load gate-gpu-qwen38-serve gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-p8b gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-mtp gate-gpu-glm5next-nvtier gate-gpu-glm5next-residency gate-gpu-glm5next-twocard: they link the arch dispatch and never open a nemotron_h file crates/model/src/arch/gemma_embedding2/** gate-ops gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-iq-sel gate-placement gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-ds41-bind gate-ds41-place gate-gpu-gemma-embedding2-serve gate-gpu-qwen38-nodes-remote gate-qwen3-plan gate-fixture-budget gate-gpu-gemma-embedding-e2e gate-gpu-gemma-embedding-serve EmbeddingGemma 2's header reader: no program runs it, so its code runs where a gemma-embedding2 file is described (its meta test), in bloomery-model's own unit tests, and behind the arch dispatch except gate-gpu-e2e gate-gpu-hybrid gate-gpu-p6 gate-gpu-ds41-moe gate-gpu-moe gate-gpu-p8 gate-gpu-block gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen35moe-moe gate-gpu-qwen3moe-router gate-qwen35moe-meta gate-gpu-qwen35-dspark gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-minicpm-text gate-gpu-qwen35-media gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen35moe-media gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-types gate-fixture gate-gpu-head gate-gpu-routed-legs gate-gpu-ds41-lib gate-gpu-lib gate-q38-serve-rules gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-nvtier gate-gpu-ds41-residency-a gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-long gate-gpu-ds41-chat gate-gpu-ds41-media gate-gpu-ds41-prefill gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-ds41-load gate-gpu-qwen3-serve gate-gpu-qwen38-serve gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-p8b gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-mtp gate-gpu-glm5next-nvtier gate-gpu-glm5next-residency gate-gpu-glm5next-twocard gate-gpu-glm5next-serve gate-gpu-serve-start-ds41 gate-gpu-serve-start-glm: they link the arch dispatch and never open a gemma-embedding2 file -crates/model/src/arch/nemotron_h_moe/** gate-ops gate-nemotron-h-moe-meta gate-fixture gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-iq-sel gate-placement gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-ds41-bind gate-ds41-place gate-gpu-nemotron-serve gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-qwen38-nodes-remote gate-qwen3-plan gate-fixture-budget gate-gpu-gemma-embedding-e2e gate-gpu-gemma-embedding-serve Nemotron 3.5 Bolt's header reader and fixture spec: its code runs where a nemotron_h_moe file is described (its meta test), where its fixture is planned and verified (the fixture generator's gate), in bloomery-model's own unit tests, and behind the arch dispatch and, through its plan (place::read), in bloomery-serve's nemotron seat, which gate-gpu-nemotron-serve and gate-gpu-nemotron-h-moe-serve start and every serve gate's bin links except gate-gpu-p6 gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35moe-mtp gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-gpu-clef-hidden gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-head gate-gpu-lib gate-gpu-ds41-lib gate-q38-serve-rules gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-nvtier gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-mtp gate-gpu-glm5next-nvtier gate-gpu-glm5next-twocard gate-gpu-glm5next-residency: they link the arch dispatch and never open a nemotron_h_moe file +crates/model/src/arch/nemotron_h_moe/** gate-ops gate-nemotron-h-moe-meta gate-fixture gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-iq-sel gate-placement gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-ds41-bind gate-ds41-place gate-gpu-nemotron-serve gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-qwen38-nodes-remote gate-qwen3-plan gate-fixture-budget gate-gpu-gemma-embedding-e2e gate-gpu-gemma-embedding-serve gate-nemotron-h-moe-host Nemotron 3.5 Bolt's header reader and fixture spec: its code runs where a nemotron_h_moe file is described (its meta test), where its fixture is planned and verified (the fixture generator's gate), in bloomery-model's own unit tests, and behind the arch dispatch and, through its plan (place::read), in bloomery-serve's nemotron seat, which gate-gpu-nemotron-serve and gate-gpu-nemotron-h-moe-serve start and every serve gate's bin links except gate-gpu-p6 gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35moe-mtp gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-gpu-clef-hidden gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-head gate-gpu-lib gate-gpu-ds41-lib gate-q38-serve-rules gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-nvtier gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-mtp gate-gpu-glm5next-nvtier gate-gpu-glm5next-twocard gate-gpu-glm5next-residency: they link the arch dispatch and never open a nemotron_h_moe file crates/model/src/arch/deepseek41/{hparams.rs,host.rs,kv.rs,mod.rs,names.rs,place.rs,plan.rs,roles.rs,spec.rs} gate-gpu-p4 gate-gpu-p6 gate-gpu-kquant gate-gpu-iq-sel gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-ptx-spill gate-dspark-read gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-qwen38-nodes-remote gate-qwen3-plan gate-fixture-budget gate-gpu-gemma-embedding-serve gate-gpu-gemma-embedding-e2e every gate that links an arch module, less the gguf-only kernel gates (above): a family's code runs wherever any family's file is opened except gate-gpu-block gate-q38-serve-rules gate-gpu-nvtier: they open no deepseek41 file: only model::arch's dispatch by the file's general.architecture (arch/mod.rs) reaches it crates/model/src/arch/qwen35moe/{head_list.rs,hparams.rs,host.rs,mod.rs,names.rs,place.rs,roles.rs,spec.rs} gate-gpu-p4 gate-gpu-p6 gate-gpu-kquant gate-gpu-iq-sel gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-ptx-spill gate-dspark-read gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-gpu-nvtier gate-q38-serve-rules gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-qwen38-nodes gate-gpu-nemotron-h-moe-e2e gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-qwen38-nodes-remote gate-qwen3-plan gate-fixture-budget gate-gpu-gemma-embedding-serve gate-gpu-gemma-embedding-e2e every gate that links an arch module, less the gguf-only kernel gates (above): a family's code runs wherever any family's file is opened except gate-gpu-block: they open no qwen35moe file: only model::arch's dispatch by the file's general.architecture (arch/mod.rs) reaches it crates/model/src/{fixture/**,arch/deepseek41/fixture.rs} gate-ops gate-ds41-meta gate-fixture gate-deepseek4-meta gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-ds41-load gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-qwen35moe-e2e gate-gpu-minicpm-text gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-qwen38-nodes-remote gate-fixture-budget the fixture generator and V4.1's FixtureSpec: no file but bin fixture, the fixture test and arch/deepseek41/fixture.rs names crate::fixture (grep), so the gates are the fixture gate and the --lib runs that execute arch::deepseek41::fixture's own unit tests (gate-ops unfiltered, gate-ds41-meta and gate-deepseek4-meta filtered arch::deepseek41) crates/model/src/arch/{coverage.rs,qwen35moe/mtp.rs} gate-gpu-iq-sel gate-gpu-e2e gate-ops gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-glm5next-gemm gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-app gate-gpu-lib gate-gpu-gates-lib gate-ds41-plan gate-gpu-ds41-lib gate-ds41-bind gate-ds41-place gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-ptx-spill gate-deepseek4-meta gate-glm5next-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-gpu-glm-sel gate-fixture gate-q38-serve-rules gate-gpu-nvtier gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-p6 gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-gpu-head gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-qwen38-nodes gate-gpu-nemotron-h-moe-e2e gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-qwen38-nodes-remote gate-qwen3-plan gate-fixture-budget gate-gpu-gemma-embedding-e2e gate-gpu-gemma-embedding-serve coverage::check's and qwen35moe's mtp readers: deepseek41/glm5next/qwen35moe place.rs (unimplemented, super::mtp), gpu-gates' engine.rs, gpu's qwen3moe plan/plan38/body38 (turns_as_neox, .unimplemented()) and the meta tests that call coverage::check — every gate that opens or plans those families' files; the two modules' lists coincide -crates/model/src/arch/names.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-q38-serve-rules gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-lib gate-gpu-glm5next-nvtier gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-qwen3moe-e2e gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-p6 gate-gpu-e2e gate-gpu-hybrid gate-gpu-iq-sel gate-gpu-ds41-moe gate-gpu-moe gate-gpu-p8 gate-gpu-block gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen3moe-router gate-placement gate-qwen35moe-meta gate-gpu-clef-hidden gate-gpu-qwen35-dspark gate-gpu-qwen35moe-moe gate-gpu-gemm gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-experts gate-ds41-host gate-fixture gate-union gate-gpu-ds41-comp gate-gpu-head gate-gpu-mcol gate-ds41-bind gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-place gate-gpu-ds41-chain-glue gate-gpu-ds41-engram gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-twocard gate-gpu-ds41-draft gate-gpu-ds41-long gate-gpu-ds41-nvtier gate-gpu-ds41-dspark-loop gate-gpu-ds41-media gate-gpu-ds41-prefill gate-ds41-load gate-gpu-ds41-chat gate-gpu-p8b gate-dspark-read gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-qwen35moe-mtp gate-app gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower gate-gpu-qwen38-nodes-remote gate-qwen3-plan gate-gpu-gemma4a gate-fixture-budget gate-gpu-gemma-embedding-e2e gate-gpu-gemma-embedding-serve the routed stacks' common names (routed_stacks): the GLM, Qwen3.6 and Qwen3.8 stack tables (swap.rs, qwen3moe/swap.rs for swap35.rs and swap38.rs) and the GLM card check (glm5next_card.rs) call it — every gate that opens those bodies or runs that check -crates/model/src/arch/hybrid.rs gate-gpu-p6 gate-gpu-nemotron-e2e gate-gpu-nemotron-h-moe-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-iq-sel gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-placement gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35moe-mtp gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-gpu-clef-hidden gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-gemma-embedding2-serve gate-gpu-gemma-embedding2-e2e gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-fixture gate-gpu-head gate-gpu-lib gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-nvtier gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-nodes gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-mtp gate-gpu-glm5next-nvtier gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-gemma-embedding-e2e gate-gpu-gemma-embedding-serve gate-fixture-budget gate-qwen3-plan gate-gpu-qwen38-nodes-remote the Nemotron-H families' common owner: the plan the serve seat and the body's open take (PlanInputs::plan, plan_with), and TOOLS and the spec helpers every gate that reads a spec links (arch::spec, nemotron_h_moe::spec::read) -crates/model/src/arch/mod.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-cand gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-ptx-spill gate-dspark-read gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-vision gate-deepseek4-meta gate-glm5next-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-q38-serve-rules gate-gpu-xstream gate-gpu-mimo2-flash gate-gpu-nvtier gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-minicpm-text gate-gpu-mamba2 gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower gate-gpu-gemma-embedding-head256 gate-gpu-qwen38-nodes-remote gate-qwen3-plan gate-gpu-gemma4a gate-fixture-budget gate-gpu-gemma-embedding-e2e gate-gpu-gemma-embedding-serve the dispatch and Arch: every gate that links it, the gguf-only kernel gates too — bloomery-gpu's fault sites (fault.rs) and bloomery-gpu-gates' family check (lib.rs, the oracle) read Arch in them +crates/model/src/arch/names.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-q38-serve-rules gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-lib gate-gpu-glm5next-nvtier gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-qwen3moe-e2e gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-p6 gate-gpu-e2e gate-gpu-hybrid gate-gpu-iq-sel gate-gpu-ds41-moe gate-gpu-moe gate-gpu-p8 gate-gpu-block gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen3moe-router gate-placement gate-qwen35moe-meta gate-gpu-clef-hidden gate-gpu-qwen35-dspark gate-gpu-qwen35moe-moe gate-gpu-gemm gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-experts gate-ds41-host gate-fixture gate-union gate-gpu-ds41-comp gate-gpu-head gate-gpu-mcol gate-ds41-bind gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-place gate-gpu-ds41-chain-glue gate-gpu-ds41-engram gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-twocard gate-gpu-ds41-draft gate-gpu-ds41-long gate-gpu-ds41-nvtier gate-gpu-ds41-dspark-loop gate-gpu-ds41-media gate-gpu-ds41-prefill gate-ds41-load gate-gpu-ds41-chat gate-gpu-p8b gate-dspark-read gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-qwen35moe-mtp gate-app gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower gate-gpu-qwen38-nodes-remote gate-qwen3-plan gate-gpu-gemma4a gate-fixture-budget gate-gpu-gemma-embedding-e2e gate-gpu-gemma-embedding-serve gate-nemotron-h-moe-host the routed stacks' common names (routed_stacks): the GLM, Qwen3.6 and Qwen3.8 stack tables (swap.rs, qwen3moe/swap.rs for swap35.rs and swap38.rs) and the GLM card check (glm5next_card.rs) call it — every gate that opens those bodies or runs that check +crates/model/src/arch/hybrid.rs gate-gpu-p6 gate-gpu-nemotron-e2e gate-gpu-nemotron-h-moe-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-iq-sel gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-placement gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35moe-mtp gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-gpu-clef-hidden gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-gemma-embedding2-serve gate-gpu-gemma-embedding2-e2e gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-fixture gate-gpu-head gate-gpu-lib gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-nvtier gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-nodes gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-mtp gate-gpu-glm5next-nvtier gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-gemma-embedding-e2e gate-gpu-gemma-embedding-serve gate-fixture-budget gate-qwen3-plan gate-gpu-qwen38-nodes-remote gate-nemotron-h-moe-host the Nemotron-H families' common owner: the plan the serve seat and the body's open take (PlanInputs::plan, plan_with), and TOOLS and the spec helpers every gate that reads a spec links (arch::spec, nemotron_h_moe::spec::read) +crates/model/src/arch/mod.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-cand gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-ptx-spill gate-dspark-read gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-vision gate-deepseek4-meta gate-glm5next-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-q38-serve-rules gate-gpu-xstream gate-gpu-mimo2-flash gate-gpu-nvtier gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-minicpm-text gate-gpu-mamba2 gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower gate-gpu-gemma-embedding-head256 gate-gpu-qwen38-nodes-remote gate-qwen3-plan gate-gpu-gemma4a gate-fixture-budget gate-gpu-gemma-embedding-e2e gate-gpu-gemma-embedding-serve gate-nemotron-h-moe-host the dispatch and Arch: every gate that links it, the gguf-only kernel gates too — bloomery-gpu's fault sites (fault.rs) and bloomery-gpu-gates' family check (lib.rs, the oracle) read Arch in them crates/gpu/src/weights.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-iq-sel gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-qwen35moe-attn gate-gpu-glm5next-gemm gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-app gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-ds41-place gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-ptx-spill gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-q38-serve-rules gate-gpu-nvtier gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower gate-gpu-qwen38-nodes-remote gate-qwen3-plan gate-gpu-gemma4a gate-gpu-gemma-embedding-e2e gate-gpu-gemma-embedding-serve the card weight loaders: the gates whose own targets name weights, or reach it through an open path (gpu's arch bodies -> model.rs -> host/swap_source.rs, gpu-deepseek41's body/chain/dense/experts/draft, gpu-glm5next's body/ffn; gate-gpu-xstream names GpuModel), less the synthetic kernel gates whose bins build blocks and name no weights path # crates/gpu's per-arch bodies: GpuModel is generic; a body's impls run where B is that body. @@ -146,21 +146,21 @@ crates/gpu-gates/src/q38_fixture.rs gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp crates/gpu-gates/src/q36_fixture.rs gate-gpu-qwen35moe-e2e gate-gpu-minicpm-text the Qwen3.6 body's per-layer launch counts from the header, which gate_qwen35moe_e2e reads, and which the MiniCPM text recipe runs through that bin crates/gpu-gates/src/act_rule.rs gate-gpu-iq gate-gpu-ds41-hc gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-minicpm-text gate-gpu-ds41-index gate-gpu-ds41-comp gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-chain-attn gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-gpu-ds41-step gate-gpu-dspark-experts gate-gpu-gemm gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-gemma-embedding2-e2e gate-gpu-gemma4v-tower gate-gpu-gemma4a gate-gpu-gemma-embedding-e2e the activation rules: the MoE, chain, index, comp, hc, iq, gemm and dspark experts bins that name them, and the fixture bands' readers of site_rel (shared/glm5next_tier.rs, the qwen4exp e2e and MTP bins, ds41-step's head; the qwen3moe experts, down and e2e bins and the qwen35moe e2e bin through ik_q8_2's non-finite check) crates/gpu-gates/src/flip.rs gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-ds41-step gate-gpu-glm5next-mtp gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-h-moe-e2e gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-qwen38-residency gate-gpu-gemma-embedding2-e2e gate-gpu-gemma4v-tower gate-gpu-gemma4a gate-gpu-gemma-embedding-e2e Flip, which shared/e2e.rs, the qwen4exp MTP bin, ds41-step's head cap and the GLM MTP replay's margin cap read -crates/model/src/lib.rs gate-gpu-p4 gate-gpu-p6 gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower gate-gpu-qwen38-nodes-remote gate-qwen3-plan gate-gpu-gemma4a gate-fixture-budget gate-gpu-gemma-embedding-e2e gate-gpu-gemma-embedding-serve ModelError, which every model reader returns -crates/model/src/moe.rs gate-gpu-p4 gate-gpu-p6 gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower gate-gpu-qwen38-nodes-remote gate-qwen3-plan gate-gpu-gemma4a gate-fixture-budget gate-gpu-gemma-embedding-e2e gate-gpu-gemma-embedding-serve TierError in ModelError (lib.rs), and the host expert legs the bodies and tiers call -crates/model/src/ops.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-app gate-prompts gate-gpu-lib gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-mamba2 gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower gate-gpu-qwen38-nodes-remote gate-qwen3-plan gate-gpu-gemma4a gate-gpu-gemma-embedding-e2e gate-gpu-gemma-embedding-serve DEFER_MAX_COLS (gpu's host/step.rs and host/tier.rs), Tensor2 and the matmul dispatch +crates/model/src/lib.rs gate-gpu-p4 gate-gpu-p6 gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower gate-gpu-qwen38-nodes-remote gate-qwen3-plan gate-gpu-gemma4a gate-fixture-budget gate-gpu-gemma-embedding-e2e gate-gpu-gemma-embedding-serve gate-nemotron-h-moe-host ModelError, which every model reader returns +crates/model/src/moe.rs gate-gpu-p4 gate-gpu-p6 gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower gate-gpu-qwen38-nodes-remote gate-qwen3-plan gate-gpu-gemma4a gate-fixture-budget gate-gpu-gemma-embedding-e2e gate-gpu-gemma-embedding-serve gate-nemotron-h-moe-host TierError in ModelError (lib.rs), and the host expert legs the bodies and tiers call +crates/model/src/ops.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-app gate-prompts gate-gpu-lib gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-mamba2 gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower gate-gpu-qwen38-nodes-remote gate-qwen3-plan gate-gpu-gemma4a gate-gpu-gemma-embedding-e2e gate-gpu-gemma-embedding-serve gate-nemotron-h-moe-host DEFER_MAX_COLS (gpu's host/step.rs and host/tier.rs), Tensor2 and the matmul dispatch crates/model/src/embed.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-ds41-lib gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-qwen3-serve gate-ds41-load gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-routed-legs gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-e2e gate-q38-serve-rules gate-gpu-qwen38-serve gate-gpu-gemma-embedding2-serve gate-gpu-gemma-embedding-e2e gate-gpu-gemma-embedding-serve RowLayout and EmbedTable: the V4.1 glue, the draft and the GLM body crates/model/src/placement/workstation.rs gate-gpu-iq-sel gate-placement gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-fixture gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-app gate-gpu-ds41-chain-ffn gate-gpu-ds41-lib gate-gpu-e2e gate-gpu-hybrid gate-gpu-moe gate-gpu-p10 gate-gpu-p8 gate-gpu-p8b gate-gpu-swap gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower gate-gpu-qwen38-nodes-remote gate-qwen3-plan gate-gpu-gemma4a gate-fixture-budget gate-gpu-gemma-embedding-e2e gate-gpu-gemma-embedding-serve host_available, which GpuModel's placement (gpu's model.rs) and the qwen35moe plan read crates/model/src/arch/qwen35moe/fixture.rs gate-fixture gate-gpu-gemma-embedding2-e2e gate-fixture-budget the Qwen3.8 fixture spec, which the fixture test reads crates/model/data/mtp-head-rows-qwen38-*.txt gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-mtp gate-q38-serve-rules gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-qwen3-plan gate-gpu-gemma-embedding-serve Qwen3.8's shipped MTP head list, which arch/qwen35moe/head_list.rs includes (include_str!): the gates whose targets run its reader (head_rows_of, SHIPPED) tools/fixture-variants.tsv gate-fixture gate-gpu-gemma-embedding2-e2e gate-fixture-budget the fixture variants' table, which fixture/variant.rs reads by path at run time: the fixture test -crates/models/src/lib.rs gate-gpu-p4 gate-gpu-p6 gate-gpu-iq-sel gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-runtime gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-dspark-read gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-nemotron-meta gate-nemotron-h-moe-meta gate-gpu-qwen35moe-attn gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gemma-embedding-meta gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower gate-gpu-qwen38-nodes-remote gate-qwen3-plan gate-gpu-gemma4a gate-fixture-budget gate-gpu-gemma-embedding-e2e gate-gpu-gemma-embedding-serve its re-exports (Role, Need: every placement path names models::Role through them) and ModelSpec and Score, which the arch readers and models::shape read +crates/models/src/lib.rs gate-gpu-p4 gate-gpu-p6 gate-gpu-iq-sel gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-runtime gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-dspark-read gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-nemotron-meta gate-nemotron-h-moe-meta gate-gpu-qwen35moe-attn gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gemma-embedding-meta gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower gate-gpu-qwen38-nodes-remote gate-qwen3-plan gate-gpu-gemma4a gate-fixture-budget gate-gpu-gemma-embedding-e2e gate-gpu-gemma-embedding-serve gate-nemotron-h-moe-host its re-exports (Role, Need: every placement path names models::Role through them) and ModelSpec and Score, which the arch readers and models::shape read crates/models/src/shape.rs gate-gpu-p6 gate-gpu-iq-sel gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-ops gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-e2e gate-fixture gate-gpu-head gate-gpu-lib gate-ds41-plan gate-gpu-ds41-lib gate-q38-serve-rules gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-ds41-bind gate-ds41-place gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-placement gate-nemotron-meta gate-nemotron-h-moe-meta gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gemma-embedding-meta gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-qwen38-nodes-remote gate-gpu-gates-lib gate-ds41-oracle gate-ds41-kld gate-qwen3-plan gate-fixture-budget gate-gpu-gemma-embedding-e2e gate-gpu-gemma-embedding-serve MoeShape and the router rows (RouterInst, router_row) the routers' kernels select by -crates/gguf/src/lib.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-cand gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-qdot gate-engram gate-tokenizer gate-decision-clef gate-decision-lev gate-serve gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-vision gate-gpu-vision gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-refset gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-models gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-minicpm-text gate-nemotron-meta gate-nemotron-h-moe-meta gate-gpu-mamba2 gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gemma-embedding-meta gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-emb2-mel gate-gpu-gemma4v-tower gate-gpu-gemma-embedding-head256 gate-gpu-qwen38-nodes-remote gate-qwen3-plan gate-gpu-gemma4a gate-fixture-budget gate-gpu-gemma-embedding-e2e gate-gpu-gemma-embedding-serve LoadError (in GpuError and placement's errors) and Gguf: every gate that reads a file or names a load error -crates/gguf/src/quant.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-cand gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-qdot gate-engram gate-tokenizer gate-decision-clef gate-decision-lev gate-serve gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-refset gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-vision gate-gpu-vision gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-dspark-read gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-models gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-hf gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-minicpm-text gate-nemotron-meta gate-nemotron-h-moe-meta gate-gpu-mamba2 gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gemma-embedding-meta gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-emb2-mel gate-gpu-gemma4v-tower gate-gpu-gemma-embedding-head256 gate-gpu-qwen38-nodes-remote gate-qwen3-plan gate-gpu-gemma4a gate-fixture-budget gate-gpu-gemma-embedding-e2e gate-gpu-gemma-embedding-serve GgmlType (gguf's lib.rs, placement) and the f16 conversion the card's fused kernels name; gate-dspark-read also runs gguf's own quant tests -crates/gguf/src/iq_tables.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-kquant gate-gpu-hc-gated gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-ds41-meta gate-qwen3moe-meta gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-gemm gate-gpu-glm5next-gemm gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-qdot gate-decision-clef gate-decision-lev gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-nemotron-meta gate-nemotron-h-moe-meta gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gemma-embedding-meta gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower gate-gpu-vision gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-gpu-glm5v-tower gate-gpu-qwen38-nodes-remote gate-qwen3-plan gate-gpu-gemma4a gate-fixture-budget gate-gpu-gemma-embedding-e2e gate-gpu-gemma-embedding-serve KVALUES_IQ4NL, which quant.rs's dequant and the card's IQ kernels read -crates/qdot/src/lib.rs gate-gpu-p4 gate-gpu-p6 gate-gpu-kquant gate-gpu-iq-sel gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-qwen35moe-attn gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-qdot gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-nemotron-meta gate-nemotron-h-moe-meta gate-gpu-mamba2 gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gemma-embedding-meta gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower gate-gpu-qwen38-nodes-remote gate-qwen3-plan gate-gpu-gemma4a gate-fixture-budget gate-gpu-gemma-embedding-e2e gate-gpu-gemma-embedding-serve QdotError in ModelError, and the fused dots coverage and quant ask for -crates/threads/src/lib.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-ds41-meta gate-qwen3moe-meta gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-glm5next-gemm gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-threads gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-ds41-plan gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-nemotron-meta gate-nemotron-h-moe-meta gate-gpu-p1 gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gemma-embedding-meta gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower gate-gpu-qwen38-nodes-remote gate-qwen3-plan gate-gpu-gemma4a gate-gpu-gemma-embedding-e2e gate-gpu-gemma-embedding-serve the pool: upload's memcpy_pool, ops' matmul and the host tier's legs +crates/gguf/src/lib.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-cand gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-qdot gate-engram gate-tokenizer gate-decision-clef gate-decision-lev gate-serve gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-vision gate-gpu-vision gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-refset gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-models gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-minicpm-text gate-nemotron-meta gate-nemotron-h-moe-meta gate-gpu-mamba2 gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gemma-embedding-meta gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-emb2-mel gate-gpu-gemma4v-tower gate-gpu-gemma-embedding-head256 gate-gpu-qwen38-nodes-remote gate-qwen3-plan gate-gpu-gemma4a gate-fixture-budget gate-gpu-gemma-embedding-e2e gate-gpu-gemma-embedding-serve gate-nemotron-h-moe-host LoadError (in GpuError and placement's errors) and Gguf: every gate that reads a file or names a load error +crates/gguf/src/quant.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-p5 gate-gpu-p6 gate-gpu-p9 gate-gpu-q4k-sel gate-gpu-kquant gate-gpu-cand gate-gpu-hc-gated gate-gpu-glm-router gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-qwen3moe-head gate-gpu-gemm gate-gpu-linear gate-gpu-qwen35moe-attn gate-gpu-glm-mla gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-qdot gate-engram gate-tokenizer gate-decision-clef gate-decision-lev gate-serve gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-refset gate-ds41-plan gate-gpu-ds41-rope gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ple gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-attn gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-hc gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-vision gate-gpu-vision gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-dspark-read gate-gpu-qwen3vl-tower gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5v-tower gate-models gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-hf gate-gpu-minicpmv-tower gate-gpu-vision-lib gate-gpu-minicpm-text gate-nemotron-meta gate-nemotron-h-moe-meta gate-gpu-mamba2 gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-gemma-embedding2-head512 gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gemma-embedding-meta gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-emb2-mel gate-gpu-gemma4v-tower gate-gpu-gemma-embedding-head256 gate-gpu-qwen38-nodes-remote gate-qwen3-plan gate-gpu-gemma4a gate-fixture-budget gate-gpu-gemma-embedding-e2e gate-gpu-gemma-embedding-serve gate-nemotron-h-moe-host GgmlType (gguf's lib.rs, placement) and the f16 conversion the card's fused kernels name; gate-dspark-read also runs gguf's own quant tests +crates/gguf/src/iq_tables.rs gate-gpu-p1 gate-gpu-p2 gate-gpu-p3 gate-gpu-p4 gate-gpu-kquant gate-gpu-hc-gated gate-gpu-iq gate-gpu-iq-sel gate-gpu-q6k-ids gate-gpu-ds41-hc gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-ds41-meta gate-qwen3moe-meta gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-down gate-gpu-gemm gate-gpu-glm5next-gemm gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-qdot gate-decision-clef gate-decision-lev gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-nemotron-meta gate-nemotron-h-moe-meta gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gemma-embedding-meta gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower gate-gpu-vision gate-gpu-qwen3vl-tower gate-gpu-minicpmv-tower gate-gpu-glm5v-tower gate-gpu-qwen38-nodes-remote gate-qwen3-plan gate-gpu-gemma4a gate-fixture-budget gate-gpu-gemma-embedding-e2e gate-gpu-gemma-embedding-serve gate-nemotron-h-moe-host KVALUES_IQ4NL, which quant.rs's dequant and the card's IQ kernels read +crates/qdot/src/lib.rs gate-gpu-p4 gate-gpu-p6 gate-gpu-kquant gate-gpu-iq-sel gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-placement gate-ds41-meta gate-qwen3moe-meta gate-qwen35moe-meta gate-gpu-qwen3moe-rope gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-qwen35moe-attn gate-gpu-glm-sel gate-gpu-glm5next-gemm gate-gpu-qwen3moe-flash gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-fixture gate-app gate-qdot gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-gpu-gates-lib gate-ds41-oracle gate-ds41-plan gate-gpu-ds41-comp gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-ds41-kld gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-nemotron-meta gate-nemotron-h-moe-meta gate-gpu-mamba2 gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gemma-embedding-meta gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower gate-gpu-qwen38-nodes-remote gate-qwen3-plan gate-gpu-gemma4a gate-fixture-budget gate-gpu-gemma-embedding-e2e gate-gpu-gemma-embedding-serve gate-nemotron-h-moe-host QdotError in ModelError, and the fused dots coverage and quant ask for +crates/threads/src/lib.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-xstream gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-ops gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-ds41-meta gate-qwen3moe-meta gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-glm5next-gemm gate-gpu-qwen3moe-e2e gate-ds41-host gate-union gate-r8 gate-threads gate-app gate-prompts gate-gpu-head gate-gpu-mcol gate-gpu-lib gate-ds41-plan gate-gpu-ds41-index gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-woa gate-gpu-ds41-chain-attn gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-kv gate-gpu-dspark-graph gate-deepseek4-meta gate-glm5next-meta gate-mimo2-meta gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-qwen4exp-meta gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-nemotron-meta gate-nemotron-h-moe-meta gate-gpu-p1 gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nvfp4 gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gemma-embedding-meta gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower gate-gpu-qwen38-nodes-remote gate-qwen3-plan gate-gpu-gemma4a gate-gpu-gemma-embedding-e2e gate-gpu-gemma-embedding-serve gate-nemotron-h-moe-host the pool: upload's memcpy_pool, ops' matmul and the host tier's legs crates/engram/src/lib.rs gate-gpu-e2e gate-gpu-hybrid gate-gpu-swap gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-chain-ffn gate-gpu-p10 gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-qwen3moe-e2e gate-app gate-gpu-lib gate-gpu-gates-lib gate-ds41-plan gate-gpu-ds41-lib gate-ds41-bind gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-engram gate-gpu-ds41-chain-glue gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower gate-gpu-qwen38-nodes-remote gate-qwen3-plan gate-gpu-gemma4a gate-gpu-gemma-embedding-e2e gate-gpu-gemma-embedding-serve EngramError (direct.rs) and Engram, which the V4.1 chain's glue holds crates/models/src/headpick.rs gate-deepseek4-meta gate-ds41-meta gate-ds41-plan gate-fixture gate-glm5next-meta gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-ds41-chat gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-twocard gate-gpu-glm5next-serve gate-gpu-iq-sel gate-gpu-lev-serve gate-gpu-nvtier gate-gpu-qwen3-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35moe-media gate-gpu-qwen38-residency gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen3moe-e2e gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-mimo2-meta gate-ops gate-q38-serve-rules gate-qwen3moe-meta gate-qwen4exp-meta gate-runtime gate-ds41-bind gate-ds41-place gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-p6 gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35-dspark gate-gpu-gemm gate-gpu-head gate-gpu-lib gate-gpu-ds41-lib gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-nvtier gate-gpu-ds41-long gate-gpu-ds41-prefill gate-gpu-ds41-media gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-mtp gate-gpu-glm5next-nvtier gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-nemotron-meta gate-nemotron-h-moe-meta gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-nemotron-h-moe-e2e gate-gpu-qwen38-nodes gate-gemma-embedding-meta gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-qwen38-nodes-remote gate-qwen3-plan gate-fixture-budget gate-gpu-gemma-embedding-e2e gate-gpu-gemma-embedding-serve models::headpick, the row-list head's reader, digest and loaded set: every gate whose load reads a head list gate-gpu-p6 gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35-dspark gate-gpu-gemm gate-gpu-head gate-gpu-lib gate-gpu-ds41-lib gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-nvtier gate-gpu-ds41-long gate-gpu-ds41-prefill gate-gpu-ds41-media gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-mtp gate-gpu-glm5next-nvtier gate-gpu-glm5next-twocard gate-gpu-glm5next-residency crates/engram/src/rows.rs gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gates-lib gate-q38-serve-rules gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-serve gate-gpu-qwen38-nodes-remote gate-qwen3-plan gate-gpu-gemma-embedding-serve SplitTable, which the Qwen3.8 body opens @@ -217,7 +217,7 @@ crates/gpu/src/cand.rs weekly-gpu-ds41-cand the candidate mask's kernels, which crates/gpu-deepseek41/src/{indexer.rs,chain/attn.rs,chain/attn/batch.rs} weekly-gpu-ds41-cand the indexer's split launcher and the select path's candidate launches, per row of words in the step and per chunk in the prompt batch crates/gpu-deepseek41/src/{body.rs,body/ced.rs,body/prefill.rs} weekly-gpu-ds41-cand the decode and pair wiring, the triangle's candidate source, and the prompt batch's kept blocks per batch set crates/gpu-gates/src/bin/{gate_deepseek41_long.rs,gate_deepseek41_prefill.rs,shared/ds41_cand.rs} weekly-gpu-ds41-cand the weekly's two clauses and their candidate rule -crates/placement/src/** gate-app gate-deepseek4-meta gate-ds41-bind gate-ds41-host gate-ds41-kld gate-ds41-load gate-ds41-meta gate-ds41-oracle gate-ds41-plan gate-dspark-read gate-fixture gate-glm5next-meta gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-chat gate-gpu-ds41-comp gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-engram gate-gpu-ds41-index gate-gpu-ds41-lib gate-gpu-ds41-long gate-gpu-ds41-prefill gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-dspark-experts gate-gpu-dspark-graph gate-gpu-dspark-hc gate-gpu-dspark-kv gate-gpu-e2e gate-gpu-gates-lib gate-gpu-gemm gate-gpu-glm-sel gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-lib gate-gpu-mcol gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-qwen35moe-moe gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-head gate-gpu-qwen3moe-router gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-ops gate-placement gate-qwen35moe-meta gate-qwen3moe-meta gate-qwen4exp-meta gate-union gate-gpu-p4 gate-gpu-p6 gate-gpu-iq-sel gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-block gate-gpu-p10 gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-gpu-qwen3moe-rope gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-r8 gate-prompts gate-gpu-head gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-woa gate-gpu-ds41-media gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-p8b gate-mimo2-meta gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-nemotron-meta gate-gpu-qwen35moe-attn gate-gpu-p1 gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-p2 gate-nemotron-h-moe-meta gate-gpu-nvfp4 gate-gpu-qwen38-nodes gate-gemma-embedding-meta gate-gpu-nemotron-h-moe-e2e gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower gate-gpu-qwen38-nodes-remote gate-qwen3-plan gate-gpu-gemma4a gate-fixture-budget gate-gpu-gemma-embedding-e2e gate-gpu-gemma-embedding-serve the placement planner, which bloomery-model re-exports as model::placement: every gate that plans a placement, reads a plan's figures or checks the r8 sidecar's refusal type +crates/placement/src/** gate-app gate-deepseek4-meta gate-ds41-bind gate-ds41-host gate-ds41-kld gate-ds41-load gate-ds41-meta gate-ds41-oracle gate-ds41-plan gate-dspark-read gate-fixture gate-glm5next-meta gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-chain-attn gate-gpu-ds41-chain-ffn gate-gpu-ds41-chain-glue gate-gpu-ds41-chat gate-gpu-ds41-comp gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-engram gate-gpu-ds41-index gate-gpu-ds41-lib gate-gpu-ds41-long gate-gpu-ds41-prefill gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-unpack gate-gpu-dspark-experts gate-gpu-dspark-graph gate-gpu-dspark-hc gate-gpu-dspark-kv gate-gpu-e2e gate-gpu-gates-lib gate-gpu-gemm gate-gpu-glm-sel gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-mtp gate-gpu-glm5next-twocard gate-gpu-lib gate-gpu-mcol gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-qwen35moe-moe gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-experts gate-gpu-qwen3moe-head gate-gpu-qwen3moe-router gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-mtp gate-ops gate-placement gate-qwen35moe-meta gate-qwen3moe-meta gate-qwen4exp-meta gate-union gate-gpu-p4 gate-gpu-p6 gate-gpu-iq-sel gate-gpu-hybrid gate-gpu-swap gate-gpu-p0b gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-block gate-gpu-p10 gate-attn gate-ffn gate-moe gate-head gate-forward gate-kv gate-alloc gate-derived gate-mt gate-profile gate-gpu-qwen3moe-rope gate-gpu-clef-hidden gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-iqleg gate-gpu-glm5next-gemm gate-gpu-mimo2-flash gate-gpu-qwen3moe-flash gate-r8 gate-prompts gate-gpu-head gate-q38-serve-rules gate-ds41-place gate-gpu-ds41-woa gate-gpu-ds41-media gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-gpu-p8b gate-mimo2-meta gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-ds41-nvtier gate-gpu-glm5next-nvtier gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-nemotron-meta gate-gpu-qwen35moe-attn gate-gpu-p1 gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-e2e gate-gpu-nemotron-serve gate-gpu-nemotron-h-moe-serve gate-gpu-p2 gate-nemotron-h-moe-meta gate-gpu-nvfp4 gate-gpu-qwen38-nodes gate-gemma-embedding-meta gate-gpu-nemotron-h-moe-e2e gate-gpu-gemma-embedding2-e2e gate-gpu-gemma-embedding2-serve gate-gpu-gemma4v-tower gate-gpu-qwen38-nodes-remote gate-qwen3-plan gate-gpu-gemma4a gate-fixture-budget gate-gpu-gemma-embedding-e2e gate-gpu-gemma-embedding-serve gate-nemotron-h-moe-host the placement planner, which bloomery-model re-exports as model::placement: every gate that plans a placement, reads a plan's figures or checks the r8 sidecar's refusal type crates/model/src/arch/gemma_encoder.rs gate-ops gate-gemma-embedding2-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gemma-embedding-meta gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-nemotron-h-moe-e2e gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve gate-gpu-qwen38-nodes-remote gate-qwen3-plan gate-fixture-budget gate-gpu-gemma-embedding-e2e gate-gpu-gemma-embedding-serve the shared Gemma encoder header checks, tensor shapes, roles and layer description except gate-gpu-p6 gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-gpu-clef-hidden gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-fixture gate-gpu-head gate-gpu-lib gate-gpu-ds41-lib gate-q38-serve-rules gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-nvtier gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-mtp gate-gpu-glm5next-nvtier gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41: they link the arch dispatch and never open a Gemma encoder file crates/model/src/arch/gemma_embedding/** gate-ops gate-gemma-embedding-meta gate-gpu-qwen35moe-mtp gate-gpu-nemotron-serve gate-gpu-qwen38-nodes gate-gpu-gemma-embedding2-e2e gate-gpu-nemotron-h-moe-e2e gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve gate-gpu-qwen38-nodes-remote gate-qwen3-plan gate-fixture-budget gate-gpu-gemma-embedding-e2e gate-gpu-gemma-embedding-serve gate-ds41-bind gate-ds41-place EmbeddingGemma header reader and its synthetic and real-header contracts except gate-gpu-p6 gate-gpu-e2e gate-gpu-hybrid gate-gpu-p8 gate-gpu-moe gate-gpu-ds41-moe gate-gpu-ds41-chain-ffn gate-gpu-block gate-gpu-p10 gate-qwen35moe-meta gate-gpu-qwen3moe-router gate-gpu-qwen35moe-moe gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen35-dspark gate-gpu-minicpm-text gate-gpu-clef-hidden gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-clef-serve gate-gpu-lev-serve gate-gpu-qwen4exp-e2e gate-gpu-qwen4exp-iqleg gate-gpu-qwen4exp-mtp gate-gpu-gemm gate-gpu-qwen3moe-e2e gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-fixture gate-gpu-head gate-gpu-lib gate-gpu-ds41-lib gate-q38-serve-rules gate-gpu-ds41-step gate-gpu-ds41-tier gate-gpu-ds41-twocard gate-gpu-ds41-callstream gate-gpu-ds41-residency-a gate-gpu-ds41-nvtier gate-gpu-ds41-long gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-prefill gate-gpu-ds41-media gate-gpu-ds41-chat gate-gpu-nvtier gate-gpu-qwen38-residency gate-gpu-qwen38-twocard gate-gpu-qwen38-serve gate-gpu-qwen3-serve gate-ds41-load gate-gpu-p8b gate-gpu-dspark-experts gate-gpu-dspark-kv gate-gpu-dspark-graph gate-gpu-glm5next-e2e gate-gpu-glm5next-stagger gate-gpu-glm5next-media gate-gpu-glm5next-media-seat gate-gpu-glm5next-mtp gate-gpu-glm5next-nvtier gate-gpu-glm5next-twocard gate-gpu-glm5next-residency gate-gpu-glm5next-serve gate-gpu-serve-start-glm gate-gpu-serve-start-ds41: they link the arch dispatch and never open a Gemma encoder file crates/gpu-gates/src/beside.rs gate-gpu-clef-serve gate-gpu-glm5next-serve gate-gpu-lev-serve gate-gpu-qwen35moe-e2e gate-gpu-qwen35moe-verify gate-gpu-qwen38-serve gate-gpu-qwen38-twocard gate-gpu-qwen3-serve gate-gpu-qwen3moe-e2e gate-gpu-clef-hidden gate-gpu-qwen35-media gate-gpu-qwen35moe-media gate-gpu-serve-start-glm gate-gpu-serve-start-ds41 gate-gpu-qwen3moe-types gate-gpu-routed-legs gate-gpu-minicpm-text gate-gpu-ds41-draft gate-gpu-ds41-dspark-loop gate-gpu-ds41-chat gate-gpu-glm5next-twocard gate-gpu-nemotron-serve gate-q38-serve-rules gate-gpu-nemotron-h-moe-serve gate-gpu-gemma-embedding2-serve gate-qwen3-plan gate-gpu-gemma4a gate-gpu-gemma-embedding-serve the reserves a plan holds beside its own tensors (the image encoder's tower in a placed plan and a whole-fit verdict): the qwen3 placement every Qwen3 bin includes, the V4.1 and GLM placements, and the servers that bind the seats that name a reserve