diff --git a/crates/gpu-gates/src/bin/gate_deepseek41_prefill.rs b/crates/gpu-gates/src/bin/gate_deepseek41_prefill.rs index 98e0367b8..96a8506e7 100644 --- a/crates/gpu-gates/src/bin/gate_deepseek41_prefill.rs +++ b/crates/gpu-gates/src/bin/gate_deepseek41_prefill.rs @@ -50,9 +50,11 @@ //! still carries the ids as they came), with ids inside it raise nothing. //! The same for `ds41_ffn_handoff_8` through the launcher, which picks it //! for an 8-slot `PageLayout` (a past id at slot 7 among them), writing no -//! word of the image but its fields; a handoff of 5, 7 or 9 slots is -//! refused by name. The same for `ds41_ffn_handoff_10` over a 10-slot -//! `PageLayout` (past ids at slots 3 and 9), a handoff of 9 or 11 slots +//! word of the image but its fields; a handoff of 5 or 7 slots is +//! refused by name. The same for `ds41_ffn_handoff_9` over a 9-slot +//! `PageLayout` (past ids at slots 4 and 8), a handoff of 5, 7 or 11 slots +//! refused by name, and for `ds41_ffn_handoff_10` over a 10-slot +//! `PageLayout` (past ids at slots 3 and 9), a handoff of 11 slots //! refused by name. Then the prompt batch's 8-slot entries through their //! launchers: `ds41_ffn_card_acc_8` against the host rule //! `runtime::combine::card_sum` bit for bit (the six-slot entry against the @@ -1135,6 +1137,7 @@ mod gate { let mut ok = places_cases(gpu)?; ok &= handoff_cases(gpu)?; ok &= handoff8_cases(gpu)?; + ok &= handoff9_cases(gpu)?; ok &= handoff10_cases(gpu)?; ok &= batch8_cases(gpu)?; ok &= tile_case(gpu, w, layers, hp)?; @@ -1330,7 +1333,7 @@ mod gate { /// as they came, the weights and the activation and nothing else of the /// image, and every slot's place, the one at slot 7 included; ids at and /// past the stack (slots 3 and 7) raise `expert_id` and place [`HOST`]. - /// A handoff of 5, 7 or 9 slots is refused by name, nothing written. + /// A handoff of 5 or 7 slots is refused by name, nothing written. fn handoff8_cases(gpu: &Gpu) -> Result { let past = N_EXPERT as u32; handoff_slots_cases::( @@ -1338,16 +1341,38 @@ mod gate { [0, 3, 5, 383, 9, 12, 300, 6], [0, 3, 5, past, 9, 12, 300, past + 616], [0.25, 0.125, 0.25, 0.125, 0.0625, 0.0625, 0.0625, 0.0625], - &[5, 7, 9], + // PIN(2026-10-16): handoff8/10 stop refusing 9; the built + // nine-slot entry is checked by handoff9_cases. + &[5, 7], ) } - /// Slots a token of the handoff's third entry (Qwen3.8's). + /// Slots a token of the handoff's third entry (Bolt's). + const SLOTS9: usize = 9; + + /// [`handoff8_cases`] at nine slots: `ds41_ffn_handoff_9` into an image + /// laid out by a 9-slot `PageLayout`, ids at and past the stack at slots + /// 4 and 8 (the last); a handoff of 5, 7 or 11 slots is refused by name, + /// nothing written. + fn handoff9_cases(gpu: &Gpu) -> Result { + let past = N_EXPERT as u32; + handoff_slots_cases::( + gpu, + [0, 3, 5, 383, 9, 12, 300, 6, 200], + [0, 3, 5, 383, past, 12, 300, 6, past + 616], + [ + 0.25, 0.125, 0.125, 0.125, 0.125, 0.0625, 0.0625, 0.0625, 0.0625, + ], + &[5, 7, 11], + ) + } + + /// Slots a token of the handoff's fourth entry (Qwen3.8's). const SLOTS10: usize = 10; /// [`handoff8_cases`] at ten slots: `ds41_ffn_handoff_10` into an image /// laid out by a 10-slot `PageLayout`, ids at and past the stack at slots - /// 3 and 9 (the last); a handoff of 9 or 11 slots is refused by name, + /// 3 and 9 (the last); a handoff of 11 slots is refused by name, /// nothing written. fn handoff10_cases(gpu: &Gpu) -> Result { let past = N_EXPERT as u32; @@ -1358,7 +1383,7 @@ mod gate { [ 0.25, 0.125, 0.125, 0.125, 0.0625, 0.0625, 0.0625, 0.0625, 0.0625, 0.0625, ], - &[9, 11], + &[11], ) } @@ -1590,7 +1615,10 @@ mod gate { verdict(pass) ); } - for n_used in [5, 7, 9] { + // PIN(2026-10-16): the card sum stops refusing 9 (`CardSlots::Nine`); + // the nine-slot entry is checked by gate_hybrid's combine clauses and + // gate-gpu-lib's places test. + for n_used in [5, 7] { let z = DeviceBuffer::from_host(stream, &vec![0.0f32; SLOTS8 * N * M])?; let s = DeviceBuffer::from_host(stream, &[HOST; SLOTS8 * M])?; let mut acc = DeviceBuffer::from_host(stream, &[SENT; N * M])?; diff --git a/crates/gpu-gates/src/bin/gate_hybrid.rs b/crates/gpu-gates/src/bin/gate_hybrid.rs index 9d8eaa110..76b072f1e 100644 --- a/crates/gpu-gates/src/bin/gate_hybrid.rs +++ b/crates/gpu-gates/src/bin/gate_hybrid.rs @@ -52,6 +52,14 @@ //! The card's slot list (`card_sel`) on the same boundary: an id past the //! card's prefix is written `HOST` with the word clean, and an id past the //! whole stack is written `HOST` and raises the layer's `ExpertId`. +//! The card's routed combine (`ds41_ffn_card_acc_9`, the card slots' family +//! at nine) there, nine slots a token: with every place a card slot it is +//! `weighted_sum` bit for bit; with HOST slots (their down rows NaN, unread +//! by the contract) the kept slots give the oracle `card_sum` bit for bit — +//! fused, the fma the device compiles both kernels' `acc += w·down` to; a +//! place in `[n_card, HOST)` at its first value raises the layer's +//! `ExpertId` and NaNs every value of its token, the other tokens the +//! oracle's. //! Last, on the engine itself (`n_l` = 32): a NaN in a hybrid layer's input //! residual through `step_layer_hybrid` returns the card's fault at that //! layer — its norm's and its router's sites in the mask, printed in the @@ -107,6 +115,7 @@ fn main() -> std::process::ExitCode { #[cfg(feature = "gpu")] mod gate { + use bloomery_gpu::card_slots::{CardAcc, CardSlotKernels}; use bloomery_gpu::fused::FusedKernels; use bloomery_gpu::hybrid::{ Boundary, BoundaryShape, HOST, HostExperts, Hybrid, HybridConfig, Poison, PoisonKind, @@ -127,6 +136,7 @@ mod gate { use gguf::{GgmlType, Gguf, Split}; use model::arch::deepseek2::derived::Derived; use model::{Tensor2, Tensor2View}; + use runtime::combine::card_sum; use std::alloc::{GlobalAlloc, Layout, System}; use std::sync::atomic::{AtomicU64, Ordering}; @@ -1342,6 +1352,151 @@ mod gate { ); ok &= pass; } + + // The card's routed combine (`ds41_ffn_card_acc_9`, the card slots' + // family at nine) over this boundary's places, nine slots a token + // (Bolt's count), three tokens: with every place a card slot it is + // `weighted_sum` bit for bit — the two kernels are the same + // expression, order and contraction; with HOST slots their down + // rows hold NaN (the contract leaves them unread) and the kept + // slots still give the oracle bit for bit; a place in + // `[n_card, HOST)` names neither a card slot nor the host, so the + // layer's ExpertId is raised and every value of its token is NaN + // while the other tokens keep theirs. + { + const E: usize = 9; + const M: usize = 3; + let rows = R_HIDDEN; + let n_card = R_CARD as u32; + let w: Vec = (0..M * E) + .map(|i| 0.05 + 0.04 * ((i / E + i % E) % 7) as f32) + .collect(); + let val = + |t: usize, e: usize, d: usize| ((t * 31 + e * 17 + d * 5) % 97) as f32 * 0.25 - 3.0; + let down: Vec = (0..M * E) + .flat_map(|i| (0..rows).map(move |d| val(i / E, i % E, d))) + .collect(); + let sel_all: Vec = (0..M * E) + .map(|i| ((i / E + i % E) % R_CARD) as u32) + .collect(); + // The rule itself, as the host reads it: the shared oracle + // `runtime::combine::card_sum`, ascending slots from zero — + // fused, because the device compiles the kernels' + // `acc += w * down` (both entries, `weighted_sum` included) to + // an fma. + let sum_over = |down: &[f32], sel: &[u32], t: usize, d: usize| { + card_sum::( + std::array::from_fn(|e| down[(t * E + e) * rows + d]), + std::array::from_fn(|e| w[t * E + e]), + std::array::from_fn(|e| sel[t * E + e] < n_card), + ) + }; + let transcription = |down: &[f32], sel: &[u32]| -> Vec { + (0..M) + .flat_map(|t| (0..rows).map(move |d| sum_over(down, sel, t, d))) + .collect() + }; + let down_dev = DeviceBuffer::from_host(stream, &down)?; + let w_dev = DeviceBuffer::from_host(stream, &w)?; + let kernels = CardSlotKernels::load(gpu.context())?; + // The nine-slot card sum through the card slots' family, the + // buffers and extents the engine's card leg passes it. + let sum = + |down: &DeviceBuffer, sel: &DeviceBuffer, acc: &mut DeviceBuffer| { + kernels.enqueue_card_acc( + stream, + &CardAcc { + down, + w: &w_dev, + sel, + n: rows, + m: M, + n_card: R_CARD, + n_used: E, + w_pitch: E, + fault: gpu.layer_sink(R_LAYER)?, + }, + acc, + ) + }; + + // Every place a card slot: the card combine is the plain one, + // bit for bit, and reruns identical. + gpu.clear_fault()?; + let sel_dev = DeviceBuffer::from_host(stream, &sel_all)?; + let mut full = DeviceBuffer::::zeroed(stream, rows * M)?; + let mut card = DeviceBuffer::::zeroed(stream, rows * M)?; + gpu.elem() + .enqueue_weighted_sum(stream, &down_dev, &w_dev, rows, E as u32, M, &mut full)?; + sum(&down_dev, &sel_dev, &mut card)?; + stream.synchronize()?; + let full_y = full.to_host_vec(stream)?; + let card_y = card.to_host_vec(stream)?; + sum(&down_dev, &sel_dev, &mut card)?; + stream.synchronize()?; + let rerun = bits_equal(&card_y, &card.to_host_vec(stream)?); + let same = bits_equal(&full_y, &card_y); + let pass = same && rerun && gpu.take_fault()?.is_none(); + println!( + "refusal card_acc_9 every slot a card slot: == weighted_sum bits {same}, \ + rerun identical {rerun}, word clean {}", + verdict(pass) + ); + ok &= pass; + + // HOST slots at chosen places, their down rows NaN: the kept + // slots' sum is `card_sum` bit for bit. + gpu.clear_fault()?; + let mut down_h = down.clone(); + let mut sel_h = sel_all.clone(); + let host_slots = [&[2usize, 5][..], &[0, 8][..], &[3][..]]; + for (t, slots) in host_slots.iter().enumerate() { + for &e in *slots { + sel_h[t * E + e] = HOST; + for d in 0..rows { + down_h[(t * E + e) * rows + d] = f32::NAN; + } + } + } + let want_h = transcription(&down_h, &sel_h); + let down_h_dev = DeviceBuffer::from_host(stream, &down_h)?; + let sel_h_dev = DeviceBuffer::from_host(stream, &sel_h)?; + sum(&down_h_dev, &sel_h_dev, &mut card)?; + stream.synchronize()?; + let host_y = card.to_host_vec(stream)?; + let pass = bits_equal(&host_y, &want_h) && gpu.take_fault()?.is_none(); + println!( + "refusal card_acc_9 HOST slots skipped, their rows unread (NaN): == card_sum \ + bits {}", + verdict(pass) + ); + ok &= pass; + + // A place in [n_card, HOST), at its first value: the layer's + // ExpertId, that token every value NaN, the others the oracle. + gpu.clear_fault()?; + let mut sel_bad = sel_h.clone(); + sel_bad[E + 6] = n_card; + let want_bad = transcription(&down_h, &sel_bad); + let sel_bad_dev = DeviceBuffer::from_host(stream, &sel_bad)?; + sum(&down_h_dev, &sel_bad_dev, &mut card)?; + stream.synchronize()?; + let bad_y = card.to_host_vec(stream)?; + let token = |y: &[f32], t: usize| y[t * rows..(t + 1) * rows].to_vec(); + let bad_nan = token(&bad_y, 1).iter().all(|v| v.is_nan()); + let others = bits_equal(&token(&bad_y, 0), &token(&want_bad, 0)) + && bits_equal(&token(&bad_y, 2), &token(&want_bad, 2)); + let word = gpu.take_fault()?; + let word_ok = word == Some(Fault::at(layer, FaultSite::ExpertId)); + let pass = bad_nan && others && word_ok; + println!( + "refusal card_acc_9 a place in [n_card, HOST) at n_card: token NaN {bad_nan}, \ + other tokens the oracle {others}, word \"{}\" {}", + word.map_or_else(|| "none".to_string(), |f| f.to_string()), + verdict(pass) + ); + ok &= pass; + } stream.synchronize()?; println!("refusal verdict {}", verdict(ok)); Ok(ok) diff --git a/crates/gpu/src/card_slots.rs b/crates/gpu/src/card_slots.rs index 869c65b33..a7d42e8f5 100644 --- a/crates/gpu/src/card_slots.rs +++ b/crates/gpu/src/card_slots.rs @@ -42,6 +42,8 @@ const _: () = assert!(THREADS == 256); pub const SLOTS_6: usize = 6; /// Slots a token of the `_8` sum, whose contract spells it as a literal. pub const SLOTS_8: usize = 8; +/// Slots a token of the `_9` sum, whose contract spells it as a literal. +pub const SLOTS_9: usize = 9; /// Slots a token of the `_10` sum, whose contract spells it as a literal. pub const SLOTS_10: usize = 10; @@ -53,6 +55,8 @@ pub enum CardSlots { Six, /// Eight slots a token (GLM-5.3-Flash). Eight, + /// Nine slots a token (Bolt). + Nine, /// Ten slots a token, the router's weights at a pitch of eleven /// (Qwen3.8). Ten, @@ -65,6 +69,7 @@ impl CardSlots { match n_used { SLOTS_6 => Some(CardSlots::Six), SLOTS_8 => Some(CardSlots::Eight), + SLOTS_9 => Some(CardSlots::Nine), SLOTS_10 => Some(CardSlots::Ten), _ => None, } @@ -76,6 +81,7 @@ impl CardSlots { match self { CardSlots::Six => SLOTS_6, CardSlots::Eight => SLOTS_8, + CardSlots::Nine => SLOTS_9, CardSlots::Ten => SLOTS_10, } } @@ -86,6 +92,7 @@ impl CardSlots { match self { CardSlots::Six => "ds41_ffn_card_acc", CardSlots::Eight => "ds41_ffn_card_acc_8", + CardSlots::Nine => "ds41_ffn_card_acc_9", CardSlots::Ten => "ds41_ffn_card_acc_10", } } @@ -95,7 +102,7 @@ impl CardSlots { GpuError::Shape { what: "ds41_ffn_card_acc", detail: format!( - "{n_used} routed slots a token: the card sum's entries are built for 6, 8 and 10" + "{n_used} routed slots a token: the card sum's entries are built for 6, 8, 9 and 10" ), } } @@ -310,6 +317,48 @@ mod card_slots_kernels { // SAFETY: i < n·m <= acc.len(); thread i is acc[i]'s only writer. unsafe { *acc.get_unchecked_mut(i) = v }; } + + /// [`ds41_ffn_card_acc`] of nine slots a token ([`card_acc_at`]). + #[allow( + clippy::too_many_arguments, + reason = "kernel entry: the device ABI takes the arguments flat (rust-quality R8)" + )] + #[kernel] + #[launch_bounds(256)] + #[launch_contract( + domain = 1, + block = (256, 1, 1), + requires = ( + w_pitch >= 9, + down.len() >= 9 * n * m, + w.len() >= w_pitch * m, + sel.len() >= 9 * m, + acc.len() >= n * m + ) + )] + pub fn ds41_ffn_card_acc_9( + down: &[f32], + w: &[f32], + sel: &[u32], + n: u32, + m: u32, + n_card: u32, + w_pitch: u32, + fault: FaultSink, + mut acc: DisjointSlice, + ) { + let (n, m) = (n as usize, m as usize); + let i = thread::index_1d().get(); + if i >= n * m { + return; + } + // SAFETY: i < n·m, and the launch contract gives every length the + // helper's contract asks for at SLOTS_9 slots. + let v = + unsafe { card_acc_at::(down, w, sel, n, n_card, w_pitch as usize, fault, i) }; + // SAFETY: i < n·m <= acc.len(); thread i is acc[i]'s only writer. + unsafe { *acc.get_unchecked_mut(i) = v }; + } } /// The card slots' sum of value `i` (token `t = i / n`, value `d = i % n`) @@ -575,6 +624,12 @@ impl CardSlotKernels { stream, &prep, a.down, a.w, a.sel, n, m, n_card, w_pitch, a.fault, acc, )?; } + CardSlots::Nine => { + let prep = mo.prepare_ds41_ffn_card_acc_9(cfg)?; + mo.ds41_ffn_card_acc_9( + stream, &prep, a.down, a.w, a.sel, n, m, n_card, w_pitch, a.fault, acc, + )?; + } CardSlots::Ten => { let prep = mo.prepare_ds41_ffn_card_acc_10(cfg)?; mo.ds41_ffn_card_acc_10( @@ -605,7 +660,7 @@ mod tests { ) } - /// The sum at 6, 8 and 10 slots (the weights at the router's pitch of + /// The sum at 6, 8, 9 and 10 slots (the weights at the router's pitch of /// eleven at ten): a token whose slots are on the card or [`HOST`] is /// `runtime::combine::card_sum`'s and raises nothing; a place in /// `[n_card, HOST)` — the first past the card and the last before the @@ -631,7 +686,7 @@ mod tests { (state >> 40) as f32 / (1u64 << 24) as f32 - 0.5 }; let mut red = Vec::new(); - for n_used in [SLOTS_6, SLOTS_8, SLOTS_10] { + for n_used in [SLOTS_6, SLOTS_8, SLOTS_9, SLOTS_10] { let w_pitch = if n_used == SLOTS_10 { n_used + 1 } else { @@ -703,6 +758,7 @@ mod tests { let want = match n_used { SLOTS_6 => rule::(args.0, args.1, args.2), SLOTS_8 => rule::(args.0, args.1, args.2), + SLOTS_9 => rule::(args.0, args.1, args.2), _ => rule::(args.0, args.1, args.2), }; g.to_bits() != want.to_bits() @@ -730,7 +786,7 @@ mod tests { } // A card of no expert: every place is the host's, and the sum is // zeros with nothing raised. - for n_used in [SLOTS_6, SLOTS_8, SLOTS_10] { + for n_used in [SLOTS_6, SLOTS_8, SLOTS_9, SLOTS_10] { let sel = DeviceBuffer::from_host(stream, &vec![HOST; M * n_used]).expect("sel"); let z = DeviceBuffer::from_host(stream, &vec![f32::NAN; M * n_used * N]).expect("z"); let mut acc = DeviceBuffer::from_host(stream, &[1.0f32; N * M]).expect("acc"); diff --git a/crates/gpu/src/host/handoff.rs b/crates/gpu/src/host/handoff.rs index ec150fe76..8d73f66fa 100644 --- a/crates/gpu/src/host/handoff.rs +++ b/crates/gpu/src/host/handoff.rs @@ -10,9 +10,10 @@ //! [`FaultSite::ExpertId`] there. //! //! The entries are built per routed slot count a token — `ds41_ffn_handoff` -//! for 6 (DeepSeek-V4.1), `_8` (GLM-5.3-Flash) and `_10` (Qwen3.8), the last -//! two over one body ([`handoff_at`]) — and the launcher picks the entry from -//! the page's slot count ([`HandoffSlots::of`]), refusing any other by name. +//! for 6 (DeepSeek-V4.1), `_8` (GLM-5.3-Flash), `_9` (Bolt) and `_10` +//! (Qwen3.8), the last three over one body ([`handoff_at`]) — and the +//! launcher picks the entry from the page's slot count ([`HandoffSlots::of`]), +//! refusing any other by name. //! `ds41_ffn_handoff_10_cols` writes a row of several columns of ten slots in //! one launch ([`HandoffKernels::enqueue_handoff_cols`]); //! [`HandoffKernels::enqueue_places_cols`] writes the places alone of such a @@ -45,14 +46,16 @@ const HANDOFF_THREADS: u32 = 256; /// Slots a token of the first entry, `ds41_ffn_handoff`, whose contract /// spells it as a literal. const N_USED: usize = 6; -/// Slots a token of the `_8` and `_10` entries, whose contracts spell them -/// as literals. +/// Slots a token of the `_8`, `_9` and `_10` entries, whose contracts spell +/// them as literals. const SLOTS_8: usize = 8; +const SLOTS_9: usize = 9; const SLOTS_10: usize = 10; // The instance table's counts are the entries' own. const _: () = assert!( HandoffSlots::Six.n() == N_USED && HandoffSlots::Eight.n() == SLOTS_8 + && HandoffSlots::Nine.n() == SLOTS_9 && HandoffSlots::Ten.n() == SLOTS_10 ); @@ -64,6 +67,8 @@ pub enum HandoffSlots { Six, /// `ds41_ffn_handoff_8`. Eight, + /// `ds41_ffn_handoff_9`. + Nine, /// `ds41_ffn_handoff_10`. Ten, } @@ -78,7 +83,7 @@ impl fmt::Display for HandoffRefused { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!( f, - "{} routed slots a token: the handoff's entries are built for 6, 8 and 10", + "{} routed slots a token: the handoff's entries are built for 6, 8, 9 and 10", self.n_used ) } @@ -92,6 +97,7 @@ impl HandoffSlots { match n_used { 6 => Ok(HandoffSlots::Six), 8 => Ok(HandoffSlots::Eight), + 9 => Ok(HandoffSlots::Nine), 10 => Ok(HandoffSlots::Ten), _ => Err(HandoffRefused { n_used }), } @@ -103,6 +109,7 @@ impl HandoffSlots { match self { HandoffSlots::Six => 6, HandoffSlots::Eight => 8, + HandoffSlots::Nine => 9, HandoffSlots::Ten => 10, } } @@ -356,6 +363,71 @@ mod handoff_kernels { unsafe { handoff_tier_at::(&a, &t, d, &mut timage, &mut tsel) }; } + /// [`ds41_ffn_handoff`] of nine slots a token ([`handoff_at`]): threads + /// `s < 9` copy slot `s`'s id and weight and write its place. + #[allow( + clippy::too_many_arguments, + reason = "kernel entry: the device ABI takes the arguments flat (rust-quality R8)" + )] + #[kernel] + #[launch_bounds(256)] + #[launch_contract( + domain = 1, + block = (256, 1, 1), + requires = ( + ids_in.len() >= 9, + w_in.len() >= 9, + map.len() >= row_off + n_expert, + x.len() >= n, + seq.len() >= 1, + n >= 9, + ids_at >= seq_at + 1, + wts_at >= ids_at + 9, + x_at >= wts_at + 9, + image.len() >= x_at + n, + sel.len() >= 9 + ) + )] + pub fn ds41_ffn_handoff_9( + ids_in: &[u32], + w_in: &[f32], + map: &[u32], + row_off: u32, + n_expert: u32, + x: &[f32], + seq: &[u32], + n: u32, + seq_at: u32, + ids_at: u32, + wts_at: u32, + x_at: u32, + fault: FaultSink, + mut image: DisjointSlice, + mut sel: DisjointSlice, + ) { + let d = thread::index_1d().get(); + if d >= n as usize { + return; + } + let a = HandoffIn { + ids_in, + w_in, + map, + row_off, + n_expert, + x, + seq, + seq_at, + ids_at, + wts_at, + x_at, + fault, + }; + // SAFETY: d < n, and the launch contract is `handoff_at`'s at + // SLOTS_9 slots; thread d is the launch's only thread at d. + unsafe { handoff_at::(&a, d, &mut image, &mut sel) }; + } + /// [`ds41_ffn_handoff`] of ten slots a token ([`handoff_at`]): threads /// `s < 10` copy slot `s`'s id and weight and write its place. #[allow( @@ -874,12 +946,12 @@ impl HandoffKernels { /// Enqueue the handoff of `target.layout.n_used` slots a token /// (`ds41_ffn_handoff` for 6, `ds41_ffn_handoff_8` for 8, - /// `ds41_ffn_handoff_10` for 10; any other count is refused by name, - /// [`HandoffSlots::of`]): `h`'s routing and `target`'s activation into - /// `target`'s image, with the region's sequence word, and each slot's - /// place into `sel`. An id past the stack raises [`FaultSite::ExpertId`] - /// on `fault` and its place is [`HOST`]. One launch. Asynchronous, - /// allocation-free, capturable. + /// `ds41_ffn_handoff_9` for 9, `ds41_ffn_handoff_10` for 10; any other + /// count is refused by name, [`HandoffSlots::of`]): `h`'s routing and + /// `target`'s activation into `target`'s image, with the region's + /// sequence word, and each slot's place into `sel`. An id past the stack + /// raises [`FaultSite::ExpertId`] on `fault` and its place is [`HOST`]. + /// One launch. Asynchronous, allocation-free, capturable. pub fn enqueue_handoff( &self, stream: &CudaStream, @@ -896,6 +968,7 @@ impl HandoffKernels { let what = match slots { HandoffSlots::Six => "ds41_ffn_handoff", HandoffSlots::Eight => "ds41_ffn_handoff_8", + HandoffSlots::Nine => "ds41_ffn_handoff_9", HandoffSlots::Ten => "ds41_ffn_handoff_10", }; let n = lay.hidden; @@ -957,6 +1030,28 @@ impl HandoffKernels { sel, )?; } + HandoffSlots::Nine => { + let prep = self.module.prepare_ds41_ffn_handoff_9(cfg)?; + self.module.ds41_ffn_handoff_9( + stream, + &prep, + h.ids, + h.weights, + h.map, + row_off, + n_expert, + target.x, + target.seq, + at[0], + at[1], + at[2], + at[3], + at[4], + fault, + target.image, + sel, + )?; + } HandoffSlots::Ten => { let prep = self.module.prepare_ds41_ffn_handoff_10(cfg)?; self.module.ds41_ffn_handoff_10( @@ -1326,15 +1421,18 @@ mod tests { } #[test] - fn slots_of_takes_six_eight_and_ten_only() { + fn slots_of_takes_six_eight_nine_and_ten_only() { for n in 0..=16 { match HandoffSlots::of(n) { Ok(s) => { - assert!(n == 6 || n == 8 || n == 10, "{n} slots picked an entry"); + assert!( + n == 6 || n == 8 || n == 9 || n == 10, + "{n} slots picked an entry" + ); assert_eq!(s.n(), n); } Err(e) => { - assert!(n != 6 && n != 8 && n != 10, "{n} slots refused"); + assert!(n != 6 && n != 8 && n != 9 && n != 10, "{n} slots refused"); assert_eq!(e, HandoffRefused { n_used: n }); assert!(e.to_string().starts_with(&format!("{n} routed slots"))); } diff --git a/tools/ref/ptx-shapes.tsv b/tools/ref/ptx-shapes.tsv index 4247024b5..4117c7356 100644 --- a/tools/ref/ptx-shapes.tsv +++ b/tools/ref/ptx-shapes.tsv @@ -46,6 +46,7 @@ generate_ds41 ds41_ffn_card_acc 0 0 generate_ds41 ds41_ffn_card_acc_10 0 0 generate_ds41 ds41_ffn_card_acc_8 0 0 generate_ds41 ds41_ffn_card_acc_8_tier 0 0 +generate_ds41 ds41_ffn_card_acc_9 0 0 generate_ds41 ds41_ffn_card_acc_tier 0 0 generate_ds41 ds41_ffn_handoff 0 0 generate_ds41 ds41_ffn_handoff_10 0 0 @@ -53,6 +54,7 @@ generate_ds41 ds41_ffn_handoff_10_cols 0 0 generate_ds41 ds41_ffn_handoff_10_cols_tier 0 0 generate_ds41 ds41_ffn_handoff_8 0 0 generate_ds41 ds41_ffn_handoff_8_tier 0 0 +generate_ds41 ds41_ffn_handoff_9 0 0 generate_ds41 ds41_ffn_handoff_tier 0 0 generate_ds41 ds41_ffn_places 0 0 generate_ds41 ds41_ffn_post 0 0 @@ -369,12 +371,14 @@ gate_e2e down_add_q5_1 0 0 gate_e2e ds41_ffn_card_acc 0 0 gate_e2e ds41_ffn_card_acc_10 0 0 gate_e2e ds41_ffn_card_acc_8 0 0 +gate_e2e ds41_ffn_card_acc_9 0 0 gate_e2e ds41_ffn_handoff 0 0 gate_e2e ds41_ffn_handoff_10 0 0 gate_e2e ds41_ffn_handoff_10_cols 0 0 gate_e2e ds41_ffn_handoff_10_cols_tier 0 0 gate_e2e ds41_ffn_handoff_8 0 0 gate_e2e ds41_ffn_handoff_8_tier 0 0 +gate_e2e ds41_ffn_handoff_9 0 0 gate_e2e ds41_ffn_places 0 0 gate_e2e ds41_markov 0 0 gate_e2e embed_rows 0 0 diff --git a/tools/unsafe-ratchet.txt b/tools/unsafe-ratchet.txt index faa1ba6ac..a4043a147 100644 --- a/tools/unsafe-ratchet.txt +++ b/tools/unsafe-ratchet.txt @@ -269,7 +269,15 @@ gguf 9 2 0 # PIN(2026-10-16): gpu 1863 -> 1869 blocks, 202 -> 203 fns, new code: IQ2_XXS's block-shared decode table (fill, its # reads, the two entries' fills) replacing the grid read (bolt-iq2opt), each table access its own block with its # SAFETY line. -gpu 1869 203 10 +# PIN(2026-10-16): gpu 1869 -> 1874 blocks, 203 -> 204 fns, new code, not old code relaxed: two new kernel entries +# (bolt-b1-handoff9), ds41_ffn_handoff_9's call into handoff_at and weighted_sum_card's reads and write, and the +# split of weighted_expert_sum's body into weighted_expert_sum_keep (the plain sum's call into it), each block with +# its SAFETY line; the fn is weighted_expert_sum_keep itself. +# PIN(2026-10-17): gpu 1874 -> 1872 blocks, 204 -> 203 fns, the nine-slot card combine folded into +# the card slots' family (bolt-b1 r2): weighted_sum_card and the weighted_expert_sum_keep split +# deleted (the plain sum's body back), ds41_ffn_card_acc_9's call into card_acc_at and its write +# added, each block with its SAFETY line; the fn was the deleted keep. +gpu 1872 203 10 # PIN(2026-10-01): gpu-deepseek41 459 -> 464 blocks, 51 -> 52 fns, a new kernel entry, not old code relaxed: # ds41_ffn_card_acc_8_tier (the tier form of the card sum at 8 slots, chain/ffn/batch.rs) adds its two blocks and # card_acc_tier_at (the one unsafe fn) with its three, each with its SAFETY line.