From 65c396b355dd3d479afa81e877a603bdcd7413a3 Mon Sep 17 00:00:00 2001 From: midagedev Date: Fri, 16 Oct 2026 21:52:46 +0900 Subject: [PATCH 1/3] gpu: ds41_ffn_handoff_9 and weighted_sum_card, the host tier's nine-slot pieces (Bolt offload) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `HandoffSlots::Nine` picks the new `ds41_ffn_handoff_9` entry (`handoff_at`'s contract at nine slots; no `_9_cols` or `_9_tier`: Bolt's prompt uses the batch port, which needs no handoff launch). `weighted_sum_card` is the routed combine over the card's slots: `weighted_expert_sum`'s rule over the slots whose place is below `n_card`, the plain sum's own expression, order and contraction, so with every slot on the card the two are bit for bit equal; a place at HOST is skipped with its weight and down row unread, and a place in `[n_card, HOST)` raises `FaultSite::ExpertId` and NaNs every value of its token. elem.rs splits `weighted_expert_sum`'s loop into `weighted_expert_sum_keep` (`#[inline(always)]`, a keep predicate the plain sum holds always true), so the two entries cannot drift apart; `weighted_sum`'s PTX does not move (ptx-compare vs ce1c310ff: added exactly the two entries, nothing changed). Gates: gate_deepseek41_prefill's raise sites gain the 9 case (past ids at slots 4 and 8); the refused counts become 5, 7 and 11 — PIN(2026-10-16): 9 became a built count. gate_hybrid's refusal arm holds the combine: every slot a card slot == `weighted_sum` bit for bit; HOST slots (their down rows NaN, unread by the contract) against the fused host transcription — the device compiles both kernels' `acc += w*down` to an fma, measured (the plain mul-then-add transcription failed the first run); a place in `[n_card, HOST)` raises the layer's ExpertId and NaNs exactly its token. ptx-shapes.tsv pins both entries 0/0 (FAIL-first: unpinned red on exactly them, then green 334/334 and 267/267); the unsafe ratchet rises by the five new blocks and `weighted_expert_sum_keep` (PIN in tools/unsafe-ratchet.txt). Co-Authored-By: Claude Code --- .../src/bin/gate_deepseek41_prefill.rs | 39 +++- crates/gpu-gates/src/bin/gate_hybrid.rs | 176 ++++++++++++++++++ crates/gpu/src/elem.rs | 175 ++++++++++++++++- crates/gpu/src/host/handoff.rs | 128 +++++++++++-- tools/ref/ptx-shapes.tsv | 4 + tools/unsafe-ratchet.txt | 6 +- 6 files changed, 496 insertions(+), 32 deletions(-) diff --git a/crates/gpu-gates/src/bin/gate_deepseek41_prefill.rs b/crates/gpu-gates/src/bin/gate_deepseek41_prefill.rs index 98e0367b8..0fbc072ab 100644 --- a/crates/gpu-gates/src/bin/gate_deepseek41_prefill.rs +++ b/crates/gpu-gates/src/bin/gate_deepseek41_prefill.rs @@ -50,9 +50,11 @@ //! still carries the ids as they came), with ids inside it raise nothing. //! The same for `ds41_ffn_handoff_8` through the launcher, which picks it //! for an 8-slot `PageLayout` (a past id at slot 7 among them), writing no -//! word of the image but its fields; a handoff of 5, 7 or 9 slots is -//! refused by name. The same for `ds41_ffn_handoff_10` over a 10-slot -//! `PageLayout` (past ids at slots 3 and 9), a handoff of 9 or 11 slots +//! word of the image but its fields; a handoff of 5 or 7 slots is +//! refused by name. The same for `ds41_ffn_handoff_9` over a 9-slot +//! `PageLayout` (past ids at slots 4 and 8), a handoff of 5, 7 or 11 slots +//! refused by name, and for `ds41_ffn_handoff_10` over a 10-slot +//! `PageLayout` (past ids at slots 3 and 9), a handoff of 11 slots //! refused by name. Then the prompt batch's 8-slot entries through their //! launchers: `ds41_ffn_card_acc_8` against the host rule //! `runtime::combine::card_sum` bit for bit (the six-slot entry against the @@ -1135,6 +1137,7 @@ mod gate { let mut ok = places_cases(gpu)?; ok &= handoff_cases(gpu)?; ok &= handoff8_cases(gpu)?; + ok &= handoff9_cases(gpu)?; ok &= handoff10_cases(gpu)?; ok &= batch8_cases(gpu)?; ok &= tile_case(gpu, w, layers, hp)?; @@ -1330,7 +1333,7 @@ mod gate { /// as they came, the weights and the activation and nothing else of the /// image, and every slot's place, the one at slot 7 included; ids at and /// past the stack (slots 3 and 7) raise `expert_id` and place [`HOST`]. - /// A handoff of 5, 7 or 9 slots is refused by name, nothing written. + /// A handoff of 5 or 7 slots is refused by name, nothing written. fn handoff8_cases(gpu: &Gpu) -> Result { let past = N_EXPERT as u32; handoff_slots_cases::( @@ -1338,16 +1341,36 @@ mod gate { [0, 3, 5, 383, 9, 12, 300, 6], [0, 3, 5, past, 9, 12, 300, past + 616], [0.25, 0.125, 0.25, 0.125, 0.0625, 0.0625, 0.0625, 0.0625], - &[5, 7, 9], + &[5, 7], ) } - /// Slots a token of the handoff's third entry (Qwen3.8's). + /// Slots a token of the handoff's third entry (Bolt's). + const SLOTS9: usize = 9; + + /// [`handoff8_cases`] at nine slots: `ds41_ffn_handoff_9` into an image + /// laid out by a 9-slot `PageLayout`, ids at and past the stack at slots + /// 4 and 8 (the last); a handoff of 5, 7 or 11 slots is refused by name, + /// nothing written. + fn handoff9_cases(gpu: &Gpu) -> Result { + let past = N_EXPERT as u32; + handoff_slots_cases::( + gpu, + [0, 3, 5, 383, 9, 12, 300, 6, 200], + [0, 3, 5, 383, past, 12, 300, 6, past + 616], + [ + 0.25, 0.125, 0.125, 0.125, 0.125, 0.0625, 0.0625, 0.0625, 0.0625, + ], + &[5, 7, 11], + ) + } + + /// Slots a token of the handoff's fourth entry (Qwen3.8's). const SLOTS10: usize = 10; /// [`handoff8_cases`] at ten slots: `ds41_ffn_handoff_10` into an image /// laid out by a 10-slot `PageLayout`, ids at and past the stack at slots - /// 3 and 9 (the last); a handoff of 9 or 11 slots is refused by name, + /// 3 and 9 (the last); a handoff of 11 slots is refused by name, /// nothing written. fn handoff10_cases(gpu: &Gpu) -> Result { let past = N_EXPERT as u32; @@ -1358,7 +1381,7 @@ mod gate { [ 0.25, 0.125, 0.125, 0.125, 0.0625, 0.0625, 0.0625, 0.0625, 0.0625, 0.0625, ], - &[9, 11], + &[11], ) } diff --git a/crates/gpu-gates/src/bin/gate_hybrid.rs b/crates/gpu-gates/src/bin/gate_hybrid.rs index 9d8eaa110..801f3d099 100644 --- a/crates/gpu-gates/src/bin/gate_hybrid.rs +++ b/crates/gpu-gates/src/bin/gate_hybrid.rs @@ -52,6 +52,13 @@ //! The card's slot list (`card_sel`) on the same boundary: an id past the //! card's prefix is written `HOST` with the word clean, and an id past the //! whole stack is written `HOST` and raises the layer's `ExpertId`. +//! The card's routed combine (`weighted_sum_card`) there, nine slots a token: +//! with every place a card slot it is `weighted_sum` bit for bit; with HOST +//! slots (their down rows NaN, unread by the contract) the kept slots give +//! the host transcription of the rule bit for bit — fused, the fma the device +//! compiles both kernels' `acc += w·down` to; a place in `[n_card, HOST)` +//! raises the layer's `ExpertId` and NaNs every value of its token, the +//! other tokens the transcription. //! Last, on the engine itself (`n_l` = 32): a NaN in a hybrid layer's input //! residual through `step_layer_hybrid` returns the card's fault at that //! layer — its norm's and its router's sites in the mask, printed in the @@ -1342,6 +1349,175 @@ mod gate { ); ok &= pass; } + + // The card's routed combine (`weighted_sum_card`) over this + // boundary's places, nine slots a token (Bolt's count), three + // tokens: with every place a card slot it is `weighted_sum` bit for + // bit — the two kernels are the same expression, order and + // contraction; with HOST slots their down rows hold NaN (the + // contract leaves them unread) and the kept slots still give the + // host transcription of the rule bit for bit; a place in + // `[n_card, HOST)` names neither a card slot nor the host, so the + // layer's ExpertId is raised and every value of its token is NaN + // while the other tokens keep theirs. + { + const E: usize = 9; + const M: usize = 3; + let rows = R_HIDDEN; + let n_card = R_CARD as u32; + let w: Vec = (0..M * E) + .map(|i| 0.05 + 0.04 * ((i / E + i % E) % 7) as f32) + .collect(); + let val = + |t: usize, e: usize, d: usize| ((t * 31 + e * 17 + d * 5) % 97) as f32 * 0.25 - 3.0; + let down: Vec = (0..M * E) + .flat_map(|i| (0..rows).map(move |d| val(i / E, i % E, d))) + .collect(); + let sel_all: Vec = (0..M * E) + .map(|i| ((i / E + i % E) % R_CARD) as u32) + .collect(); + // The rule itself, as the host reads it: ascending slots, the + // kept slots' `acc += w * down` from 0 — fused, because the + // device compiles the kernels' `acc += w * down` (both entries, + // `weighted_sum` included) to an fma. + let sum_over = |down: &[f32], sel: &[u32], t: usize, d: usize| { + let mut acc = 0.0f32; + for e in 0..E { + if sel[t * E + e] < n_card { + acc = w[t * E + e].mul_add(down[(t * E + e) * rows + d], acc); + } + } + acc + }; + let transcription = |down: &[f32], sel: &[u32]| -> Vec { + (0..M) + .flat_map(|t| (0..rows).map(move |d| sum_over(down, sel, t, d))) + .collect() + }; + let down_dev = DeviceBuffer::from_host(stream, &down)?; + let w_dev = DeviceBuffer::from_host(stream, &w)?; + + // Every place a card slot: the card combine is the plain one, + // bit for bit, and reruns identical. + gpu.clear_fault()?; + let sel_dev = DeviceBuffer::from_host(stream, &sel_all)?; + let mut full = DeviceBuffer::::zeroed(stream, rows * M)?; + let mut card = DeviceBuffer::::zeroed(stream, rows * M)?; + gpu.elem() + .enqueue_weighted_sum(stream, &down_dev, &w_dev, rows, E as u32, M, &mut full)?; + gpu.elem().enqueue_weighted_sum_card( + stream, + &down_dev, + &w_dev, + &sel_dev, + n_card, + rows, + E as u32, + M, + gpu.layer_sink(R_LAYER)?, + &mut card, + )?; + stream.synchronize()?; + let full_y = full.to_host_vec(stream)?; + let card_y = card.to_host_vec(stream)?; + gpu.elem().enqueue_weighted_sum_card( + stream, + &down_dev, + &w_dev, + &sel_dev, + n_card, + rows, + E as u32, + M, + gpu.layer_sink(R_LAYER)?, + &mut card, + )?; + stream.synchronize()?; + let rerun = bits_equal(&card_y, &card.to_host_vec(stream)?); + let same = bits_equal(&full_y, &card_y); + let pass = same && rerun && gpu.take_fault()?.is_none(); + println!( + "refusal weighted_sum_card every slot a card slot: == weighted_sum bits {same}, \ + rerun identical {rerun}, word clean {}", + verdict(pass) + ); + ok &= pass; + + // HOST slots at chosen places, their down rows NaN: the kept + // slots' sum is the transcription bit for bit. + gpu.clear_fault()?; + let mut down_h = down.clone(); + let mut sel_h = sel_all.clone(); + let host_slots = [&[2usize, 5][..], &[0, 8][..], &[3][..]]; + for (t, slots) in host_slots.iter().enumerate() { + for &e in *slots { + sel_h[t * E + e] = HOST; + for d in 0..rows { + down_h[(t * E + e) * rows + d] = f32::NAN; + } + } + } + let want_h = transcription(&down_h, &sel_h); + let down_h_dev = DeviceBuffer::from_host(stream, &down_h)?; + let sel_h_dev = DeviceBuffer::from_host(stream, &sel_h)?; + gpu.elem().enqueue_weighted_sum_card( + stream, + &down_h_dev, + &w_dev, + &sel_h_dev, + n_card, + rows, + E as u32, + M, + gpu.layer_sink(R_LAYER)?, + &mut card, + )?; + stream.synchronize()?; + let host_y = card.to_host_vec(stream)?; + let pass = bits_equal(&host_y, &want_h) && gpu.take_fault()?.is_none(); + println!( + "refusal weighted_sum_card HOST slots skipped, their rows unread (NaN): == the \ + host transcription bits {}", + verdict(pass) + ); + ok &= pass; + + // A place in [n_card, HOST): the layer's ExpertId, that token + // every value NaN, the others the transcription. + gpu.clear_fault()?; + let mut sel_bad = sel_h.clone(); + sel_bad[E + 6] = n_card + 3; + let want_bad = transcription(&down_h, &sel_bad); + let sel_bad_dev = DeviceBuffer::from_host(stream, &sel_bad)?; + gpu.elem().enqueue_weighted_sum_card( + stream, + &down_h_dev, + &w_dev, + &sel_bad_dev, + n_card, + rows, + E as u32, + M, + gpu.layer_sink(R_LAYER)?, + &mut card, + )?; + stream.synchronize()?; + let bad_y = card.to_host_vec(stream)?; + let token = |y: &[f32], t: usize| y[t * rows..(t + 1) * rows].to_vec(); + let bad_nan = token(&bad_y, 1).iter().all(|v| v.is_nan()); + let others = bits_equal(&token(&bad_y, 0), &token(&want_bad, 0)) + && bits_equal(&token(&bad_y, 2), &token(&want_bad, 2)); + let word = gpu.take_fault()?; + let word_ok = word == Some(Fault::at(layer, FaultSite::ExpertId)); + let pass = bad_nan && others && word_ok; + println!( + "refusal weighted_sum_card a place in [n_card, HOST): token NaN {bad_nan}, other \ + tokens the transcription {others}, word \"{}\" {}", + word.map_or_else(|| "none".to_string(), |f| f.to_string()), + verdict(pass) + ); + ok &= pass; + } stream.synchronize()?; println!("refusal verdict {}", verdict(ok)); Ok(ok) diff --git a/crates/gpu/src/elem.rs b/crates/gpu/src/elem.rs index 5044669f1..75a6dc47e 100644 --- a/crates/gpu/src/elem.rs +++ b/crates/gpu/src/elem.rs @@ -12,7 +12,9 @@ //! `width * m` spans; `rope` on `[n_dims, n_vec, m]` (column `c = t*n_vec + //! v`, the layout of the reference dump's `q_rope`/`k_rope` views); //! `weighted_sum` on `[rows, n_exp, m]` down-projections with `[n_exp, m]` -//! weights; the embedding table is Q3_K rows of 2048 values (880 bytes = 220 +//! weights, and `weighted_sum_card` over the card's share of the same (the +//! places `[n_exp, m]` beside the weights, the host's skipped); the embedding +//! table is Q3_K rows of 2048 values (880 bytes = 220 //! u32 words each), Q3_K, Q4_K, Q5_K or Q6_K rows of whole super-blocks, or //! Q5_0 rows in the `pack_q5_0` layout. //! Extents are launch arguments, never buffer lengths: scratch buffers may @@ -543,19 +545,46 @@ pub(crate) unsafe fn weighted_expert_sum( n_exp: u32, t: usize, d: usize, +) -> f32 { + // SAFETY: every slot kept, so this fn's `# Safety` is the helper's. + unsafe { weighted_expert_sum_keep(down, w, rows, n_exp, t, d, |_| true) } +} + +/// [`weighted_expert_sum`] over the slots `keep` names: the same loop with +/// the refused slots' terms left out, their weight and down row unread. The +/// kept slots' expression, order and contraction are the plain sum's own, so +/// a caller that keeps every slot compiles to the plain sum's instructions; +/// the loop is this one helper so the two entries cannot drift apart. +/// +/// # Safety +/// +/// [`weighted_expert_sum`]'s contract, for the slots `keep` names; the +/// predicate's own reads are its caller's. +#[inline(always)] +pub(crate) unsafe fn weighted_expert_sum_keep( + down: &[f32], + w: &[f32], + rows: u32, + n_exp: u32, + t: usize, + d: usize, + mut keep: impl FnMut(usize) -> bool, ) -> f32 { let rows = rows as usize; let n_exp = n_exp as usize; let mut acc = 0.0f32; let mut e = 0usize; while e < n_exp { - // SAFETY: e < n_exp and t < m bound the weight index t*n_exp + e - // inside `w` by this fn's `# Safety`. - let wv = unsafe { *w.get_unchecked(t * n_exp + e) }; - // SAFETY: the same e < n_exp and t < m, with d < rows, bound the down - // index (t*n_exp + e)*rows + d inside `down` by this fn's `# Safety`. - let dv = unsafe { *down.get_unchecked((t * n_exp + e) * rows + d) }; - acc += wv * dv; + if keep(e) { + // SAFETY: e < n_exp and t < m bound the weight index t*n_exp + e + // inside `w` by this fn's `# Safety`. + let wv = unsafe { *w.get_unchecked(t * n_exp + e) }; + // SAFETY: the same e < n_exp and t < m, with d < rows, bound the + // down index (t*n_exp + e)*rows + d inside `down` by this fn's + // `# Safety`. + let dv = unsafe { *down.get_unchecked((t * n_exp + e) * rows + d) }; + acc += wv * dv; + } e += 1; } acc @@ -1221,6 +1250,72 @@ mod elem_kernels { } } + /// The routed-expert combine over the card's slots, one thread per + /// (token, output value): `weighted_sum`'s rule over the slots whose + /// place `sel[t*n_exp + e]` is below `n_card` — the same expression, + /// order and contraction, so with every slot on the card the two are bit + /// for bit equal. A place at [`HOST`] is the host's slot, skipped with + /// its down row and weight unread; a place in `[n_card, HOST)` names + /// neither a card slot nor the host: it raises [`FaultSite::ExpertId`] + /// on `fault` and every value of its token is NaN. + #[allow( + clippy::too_many_arguments, + reason = "kernel entry: the device ABI takes the arguments flat (rust-quality R8)" + )] + #[kernel] + #[launch_bounds(256)] + #[launch_contract( + domain = 1, + block = (256, 1, 1), + requires = ( + down.len() >= rows * n_exp * m, + w.len() >= n_exp * m, + sel.len() >= n_exp * m, + y.len() >= rows * m + ) + )] + pub fn weighted_sum_card( + down: &[f32], + w: &[f32], + sel: &[u32], + n_card: u32, + rows: u32, + n_exp: u32, + m: u32, + fault: FaultSink, + mut y: DisjointSlice, + ) { + let i = thread::index_1d().get(); + if i >= rows as usize * m as usize { + return; + } + let (t, d) = (i / rows as usize, i % rows as usize); + let mut faulted = false; + // SAFETY: i < rows·m (the guard above) gives t < m and d < rows; the + // launch contract gives down.len() >= rows·n_exp·m, w.len() >= + // n_exp·m and sel.len() >= n_exp·m. + let acc = unsafe { + weighted_expert_sum_keep(down, w, rows, n_exp, t, d, |e| { + // SAFETY: t < m and e < n_exp bound t·n_exp + e inside `sel` + // by the launch contract. + let place = unsafe { *sel.get_unchecked(t * n_exp as usize + e) }; + let keep = place < n_card; + if !keep && place != HOST { + faulted = true; + if d == 0 { + fault.raise(FaultSite::ExpertId); + } + } + keep + }) + }; + let v = if faulted { f32::NAN } else { acc }; + // SAFETY: i < rows*m <= y.len() by the launch contract. + unsafe { + *y.get_unchecked_mut(i) = v; + } + } + /// Widen `n` f16 bit patterns (one per word of `bits`, the low 16 bits) /// to f32 through [`half_to_f32`], the decode every weight scale in this /// package goes through. It exists so the gate can hold that decode @@ -2336,6 +2431,70 @@ impl ElemKernels { Ok(()) } + /// Enqueue the routed-expert combine over the card's slots: + /// `enqueue_weighted_sum`'s buffers and extents, with `sel` the places + /// (`n_exp * m` u32, `[n_exp, m]` beside `w`) and `n_card` the card's + /// slot count — a place below `n_card` is that card slot's, [`HOST`] is + /// the host's (skipped, its weight and down row unread), and a place in + /// `[n_card, HOST)` raises [`FaultSite::ExpertId`] on `fault` and NaNs + /// every value of its token. Asynchronous, allocation-free, capturable. + #[allow( + clippy::too_many_arguments, + reason = "the combine's two buffers, places, card count, extents, fault and output (rust-quality R8)" + )] + pub fn enqueue_weighted_sum_card( + &self, + stream: &CudaStream, + down: &DeviceBuffer, + w: &DeviceBuffer, + sel: &DeviceBuffer, + n_card: u32, + rows: usize, + n_exp: u32, + m: usize, + fault: FaultSink, + y: &mut DeviceBuffer, + ) -> Result<(), GpuError> { + if rows == 0 || n_exp == 0 || m == 0 { + return Err(GpuError::shape( + "enqueue_weighted_sum_card", + format!("need rows/n_exp/m >= 1, got {rows}/{n_exp}/{m}"), + )); + } + if down.len() < rows * n_exp as usize * m + || w.len() < n_exp as usize * m + || sel.len() < n_exp as usize * m + || y.len() < rows * m + { + return Err(GpuError::shape( + "enqueue_weighted_sum_card", + format!( + "down.len() {} (need {}), w.len() {} (need {}), sel.len() {} (need {}), \ + y.len() {} (need {})", + down.len(), + rows * n_exp as usize * m, + w.len(), + n_exp as usize * m, + sel.len(), + n_exp as usize * m, + y.len(), + rows * m + ), + )); + } + let what = "enqueue_weighted_sum_card"; + let grid = launch_u32(what, "grid", (rows * m).div_ceil(256))?; + let rows = launch_u32(what, "rows", rows)?; + let m = launch_u32(what, "m", m)?; + let prep = self + .module + .prepare_weighted_sum_card(LaunchConfig1D::new(grid, 256, 0))?; + self.module.weighted_sum_card( + stream, &prep, down, w, sel, n_card, rows, n_exp, m, fault, y, + )?; + Ok(()) + } + /// Enqueue the f16 widening of `bits` (the low 16 bits of each word) /// into `y`, through the same `cores::half_to_f32` the gemvs and the /// embedding dequant call. The gate's instrument, not a step op. diff --git a/crates/gpu/src/host/handoff.rs b/crates/gpu/src/host/handoff.rs index ec150fe76..8d73f66fa 100644 --- a/crates/gpu/src/host/handoff.rs +++ b/crates/gpu/src/host/handoff.rs @@ -10,9 +10,10 @@ //! [`FaultSite::ExpertId`] there. //! //! The entries are built per routed slot count a token — `ds41_ffn_handoff` -//! for 6 (DeepSeek-V4.1), `_8` (GLM-5.3-Flash) and `_10` (Qwen3.8), the last -//! two over one body ([`handoff_at`]) — and the launcher picks the entry from -//! the page's slot count ([`HandoffSlots::of`]), refusing any other by name. +//! for 6 (DeepSeek-V4.1), `_8` (GLM-5.3-Flash), `_9` (Bolt) and `_10` +//! (Qwen3.8), the last three over one body ([`handoff_at`]) — and the +//! launcher picks the entry from the page's slot count ([`HandoffSlots::of`]), +//! refusing any other by name. //! `ds41_ffn_handoff_10_cols` writes a row of several columns of ten slots in //! one launch ([`HandoffKernels::enqueue_handoff_cols`]); //! [`HandoffKernels::enqueue_places_cols`] writes the places alone of such a @@ -45,14 +46,16 @@ const HANDOFF_THREADS: u32 = 256; /// Slots a token of the first entry, `ds41_ffn_handoff`, whose contract /// spells it as a literal. const N_USED: usize = 6; -/// Slots a token of the `_8` and `_10` entries, whose contracts spell them -/// as literals. +/// Slots a token of the `_8`, `_9` and `_10` entries, whose contracts spell +/// them as literals. const SLOTS_8: usize = 8; +const SLOTS_9: usize = 9; const SLOTS_10: usize = 10; // The instance table's counts are the entries' own. const _: () = assert!( HandoffSlots::Six.n() == N_USED && HandoffSlots::Eight.n() == SLOTS_8 + && HandoffSlots::Nine.n() == SLOTS_9 && HandoffSlots::Ten.n() == SLOTS_10 ); @@ -64,6 +67,8 @@ pub enum HandoffSlots { Six, /// `ds41_ffn_handoff_8`. Eight, + /// `ds41_ffn_handoff_9`. + Nine, /// `ds41_ffn_handoff_10`. Ten, } @@ -78,7 +83,7 @@ impl fmt::Display for HandoffRefused { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!( f, - "{} routed slots a token: the handoff's entries are built for 6, 8 and 10", + "{} routed slots a token: the handoff's entries are built for 6, 8, 9 and 10", self.n_used ) } @@ -92,6 +97,7 @@ impl HandoffSlots { match n_used { 6 => Ok(HandoffSlots::Six), 8 => Ok(HandoffSlots::Eight), + 9 => Ok(HandoffSlots::Nine), 10 => Ok(HandoffSlots::Ten), _ => Err(HandoffRefused { n_used }), } @@ -103,6 +109,7 @@ impl HandoffSlots { match self { HandoffSlots::Six => 6, HandoffSlots::Eight => 8, + HandoffSlots::Nine => 9, HandoffSlots::Ten => 10, } } @@ -356,6 +363,71 @@ mod handoff_kernels { unsafe { handoff_tier_at::(&a, &t, d, &mut timage, &mut tsel) }; } + /// [`ds41_ffn_handoff`] of nine slots a token ([`handoff_at`]): threads + /// `s < 9` copy slot `s`'s id and weight and write its place. + #[allow( + clippy::too_many_arguments, + reason = "kernel entry: the device ABI takes the arguments flat (rust-quality R8)" + )] + #[kernel] + #[launch_bounds(256)] + #[launch_contract( + domain = 1, + block = (256, 1, 1), + requires = ( + ids_in.len() >= 9, + w_in.len() >= 9, + map.len() >= row_off + n_expert, + x.len() >= n, + seq.len() >= 1, + n >= 9, + ids_at >= seq_at + 1, + wts_at >= ids_at + 9, + x_at >= wts_at + 9, + image.len() >= x_at + n, + sel.len() >= 9 + ) + )] + pub fn ds41_ffn_handoff_9( + ids_in: &[u32], + w_in: &[f32], + map: &[u32], + row_off: u32, + n_expert: u32, + x: &[f32], + seq: &[u32], + n: u32, + seq_at: u32, + ids_at: u32, + wts_at: u32, + x_at: u32, + fault: FaultSink, + mut image: DisjointSlice, + mut sel: DisjointSlice, + ) { + let d = thread::index_1d().get(); + if d >= n as usize { + return; + } + let a = HandoffIn { + ids_in, + w_in, + map, + row_off, + n_expert, + x, + seq, + seq_at, + ids_at, + wts_at, + x_at, + fault, + }; + // SAFETY: d < n, and the launch contract is `handoff_at`'s at + // SLOTS_9 slots; thread d is the launch's only thread at d. + unsafe { handoff_at::(&a, d, &mut image, &mut sel) }; + } + /// [`ds41_ffn_handoff`] of ten slots a token ([`handoff_at`]): threads /// `s < 10` copy slot `s`'s id and weight and write its place. #[allow( @@ -874,12 +946,12 @@ impl HandoffKernels { /// Enqueue the handoff of `target.layout.n_used` slots a token /// (`ds41_ffn_handoff` for 6, `ds41_ffn_handoff_8` for 8, - /// `ds41_ffn_handoff_10` for 10; any other count is refused by name, - /// [`HandoffSlots::of`]): `h`'s routing and `target`'s activation into - /// `target`'s image, with the region's sequence word, and each slot's - /// place into `sel`. An id past the stack raises [`FaultSite::ExpertId`] - /// on `fault` and its place is [`HOST`]. One launch. Asynchronous, - /// allocation-free, capturable. + /// `ds41_ffn_handoff_9` for 9, `ds41_ffn_handoff_10` for 10; any other + /// count is refused by name, [`HandoffSlots::of`]): `h`'s routing and + /// `target`'s activation into `target`'s image, with the region's + /// sequence word, and each slot's place into `sel`. An id past the stack + /// raises [`FaultSite::ExpertId`] on `fault` and its place is [`HOST`]. + /// One launch. Asynchronous, allocation-free, capturable. pub fn enqueue_handoff( &self, stream: &CudaStream, @@ -896,6 +968,7 @@ impl HandoffKernels { let what = match slots { HandoffSlots::Six => "ds41_ffn_handoff", HandoffSlots::Eight => "ds41_ffn_handoff_8", + HandoffSlots::Nine => "ds41_ffn_handoff_9", HandoffSlots::Ten => "ds41_ffn_handoff_10", }; let n = lay.hidden; @@ -957,6 +1030,28 @@ impl HandoffKernels { sel, )?; } + HandoffSlots::Nine => { + let prep = self.module.prepare_ds41_ffn_handoff_9(cfg)?; + self.module.ds41_ffn_handoff_9( + stream, + &prep, + h.ids, + h.weights, + h.map, + row_off, + n_expert, + target.x, + target.seq, + at[0], + at[1], + at[2], + at[3], + at[4], + fault, + target.image, + sel, + )?; + } HandoffSlots::Ten => { let prep = self.module.prepare_ds41_ffn_handoff_10(cfg)?; self.module.ds41_ffn_handoff_10( @@ -1326,15 +1421,18 @@ mod tests { } #[test] - fn slots_of_takes_six_eight_and_ten_only() { + fn slots_of_takes_six_eight_nine_and_ten_only() { for n in 0..=16 { match HandoffSlots::of(n) { Ok(s) => { - assert!(n == 6 || n == 8 || n == 10, "{n} slots picked an entry"); + assert!( + n == 6 || n == 8 || n == 9 || n == 10, + "{n} slots picked an entry" + ); assert_eq!(s.n(), n); } Err(e) => { - assert!(n != 6 && n != 8 && n != 10, "{n} slots refused"); + assert!(n != 6 && n != 8 && n != 9 && n != 10, "{n} slots refused"); assert_eq!(e, HandoffRefused { n_used: n }); assert!(e.to_string().starts_with(&format!("{n} routed slots"))); } diff --git a/tools/ref/ptx-shapes.tsv b/tools/ref/ptx-shapes.tsv index 4247024b5..433a310c8 100644 --- a/tools/ref/ptx-shapes.tsv +++ b/tools/ref/ptx-shapes.tsv @@ -53,6 +53,7 @@ generate_ds41 ds41_ffn_handoff_10_cols 0 0 generate_ds41 ds41_ffn_handoff_10_cols_tier 0 0 generate_ds41 ds41_ffn_handoff_8 0 0 generate_ds41 ds41_ffn_handoff_8_tier 0 0 +generate_ds41 ds41_ffn_handoff_9 0 0 generate_ds41 ds41_ffn_handoff_tier 0 0 generate_ds41 ds41_ffn_places 0 0 generate_ds41 ds41_ffn_post 0 0 @@ -348,6 +349,7 @@ generate_ds41 swiglu_quant32_sel 0 0 generate_ds41 tier_fault_copy 0 0 generate_ds41 touch 0 0 generate_ds41 weighted_sum 0 0 +generate_ds41 weighted_sum_card 0 0 # PIN(2026-09-25): ds41_attn_seg_stage is the staged prefix segment pass (prefill only, never on the decode path); its ring ⧺ staging source keeps a few registers more than ds41_attn_seg, which already sits at the 128-register cap of a 512-thread block, and ptxas spills 8 bytes. ds41_attn_seg_sel_stage fits in 112 with none. # gate_e2e (--features gpu) @@ -375,6 +377,7 @@ gate_e2e ds41_ffn_handoff_10_cols 0 0 gate_e2e ds41_ffn_handoff_10_cols_tier 0 0 gate_e2e ds41_ffn_handoff_8 0 0 gate_e2e ds41_ffn_handoff_8_tier 0 0 +gate_e2e ds41_ffn_handoff_9 0 0 gate_e2e ds41_ffn_places 0 0 gate_e2e ds41_markov 0 0 gate_e2e embed_rows 0 0 @@ -619,6 +622,7 @@ gate_e2e swiglu_quant32_sel 0 0 gate_e2e tier_fault_copy 0 0 gate_e2e touch 0 0 gate_e2e weighted_sum 0 0 +gate_e2e weighted_sum_card 0 0 # gate_qwen3vl_tower (--features vision), the `vis_` entries of bloomery-gpu-vision only: V4.1's six, the Qwen3-VL tower's five, the glm5v tower's three, and MiniCPM's two gate_qwen3vl_tower vis_attn 0 0 diff --git a/tools/unsafe-ratchet.txt b/tools/unsafe-ratchet.txt index faa1ba6ac..a7e86bfd4 100644 --- a/tools/unsafe-ratchet.txt +++ b/tools/unsafe-ratchet.txt @@ -269,7 +269,11 @@ gguf 9 2 0 # PIN(2026-10-16): gpu 1863 -> 1869 blocks, 202 -> 203 fns, new code: IQ2_XXS's block-shared decode table (fill, its # reads, the two entries' fills) replacing the grid read (bolt-iq2opt), each table access its own block with its # SAFETY line. -gpu 1869 203 10 +# PIN(2026-10-16): gpu 1869 -> 1874 blocks, 203 -> 204 fns, new code, not old code relaxed: two new kernel entries +# (bolt-b1-handoff9), ds41_ffn_handoff_9's call into handoff_at and weighted_sum_card's reads and write, and the +# split of weighted_expert_sum's body into weighted_expert_sum_keep (the plain sum's call into it), each block with +# its SAFETY line; the fn is weighted_expert_sum_keep itself. +gpu 1874 204 10 # PIN(2026-10-01): gpu-deepseek41 459 -> 464 blocks, 51 -> 52 fns, a new kernel entry, not old code relaxed: # ds41_ffn_card_acc_8_tier (the tier form of the card sum at 8 slots, chain/ffn/batch.rs) adds its two blocks and # card_acc_tier_at (the one unsafe fn) with its three, each with its SAFETY line. From 3054319ba87fa354d45d3708b52e668d384ae9ca Mon Sep 17 00:00:00 2001 From: midagedev Date: Sat, 17 Oct 2026 06:07:01 +0900 Subject: [PATCH 2/3] gpu: the nine-slot card combine moves into CardSlots (review) The PR 343 review: a second card combine in elem.rs duplicates the card slots' family owner. `CardSlots::Nine` now picks `ds41_ffn_card_acc_9` (`card_acc_at`'s contract at nine slots) through `enqueue_card_acc`'s existing `CardAcc` validation; `weighted_sum_card`, `enqueue_weighted_sum_card` and the `weighted_expert_sum_keep` split are deleted and `weighted_expert_sum` keeps its original body, so elem.rs is byte for byte the base again. gate_hybrid's combine clauses call `CardSlotKernels`/`CardAcc` and judge against the shared oracle `runtime::combine::card_sum` (the hand transcription deleted); the stray place is now `n_card` itself, the first value in `[n_card, HOST)`, so the `place <= n_card` mutant fails the clause. gate_deepseek41_prefill's handoff8/10 refusal lists keep the dated reason for the nine-slot cases they stopped refusing. ptx-shapes: `weighted_sum_card`'s rows out, `ds41_ffn_card_acc_9` in at 0/0 (FAIL-first red on exactly the two rows at a wrong pin); unsafe ratchet gpu 1874 -> 1872 blocks, 204 -> 203 fns. Co-Authored-By: Claude Code --- .../src/bin/gate_deepseek41_prefill.rs | 2 + crates/gpu-gates/src/bin/gate_hybrid.rs | 143 ++++++-------- crates/gpu/src/card_slots.rs | 64 ++++++- crates/gpu/src/elem.rs | 175 +----------------- tools/ref/ptx-shapes.tsv | 4 +- tools/unsafe-ratchet.txt | 6 +- 6 files changed, 138 insertions(+), 256 deletions(-) diff --git a/crates/gpu-gates/src/bin/gate_deepseek41_prefill.rs b/crates/gpu-gates/src/bin/gate_deepseek41_prefill.rs index 0fbc072ab..9bf44dbc7 100644 --- a/crates/gpu-gates/src/bin/gate_deepseek41_prefill.rs +++ b/crates/gpu-gates/src/bin/gate_deepseek41_prefill.rs @@ -1341,6 +1341,8 @@ mod gate { [0, 3, 5, 383, 9, 12, 300, 6], [0, 3, 5, past, 9, 12, 300, past + 616], [0.25, 0.125, 0.25, 0.125, 0.0625, 0.0625, 0.0625, 0.0625], + // PIN(2026-10-16): handoff8/10 stop refusing 9; the built + // nine-slot entry is checked by handoff9_cases. &[5, 7], ) } diff --git a/crates/gpu-gates/src/bin/gate_hybrid.rs b/crates/gpu-gates/src/bin/gate_hybrid.rs index 801f3d099..76b072f1e 100644 --- a/crates/gpu-gates/src/bin/gate_hybrid.rs +++ b/crates/gpu-gates/src/bin/gate_hybrid.rs @@ -52,13 +52,14 @@ //! The card's slot list (`card_sel`) on the same boundary: an id past the //! card's prefix is written `HOST` with the word clean, and an id past the //! whole stack is written `HOST` and raises the layer's `ExpertId`. -//! The card's routed combine (`weighted_sum_card`) there, nine slots a token: -//! with every place a card slot it is `weighted_sum` bit for bit; with HOST -//! slots (their down rows NaN, unread by the contract) the kept slots give -//! the host transcription of the rule bit for bit — fused, the fma the device -//! compiles both kernels' `acc += w·down` to; a place in `[n_card, HOST)` -//! raises the layer's `ExpertId` and NaNs every value of its token, the -//! other tokens the transcription. +//! The card's routed combine (`ds41_ffn_card_acc_9`, the card slots' family +//! at nine) there, nine slots a token: with every place a card slot it is +//! `weighted_sum` bit for bit; with HOST slots (their down rows NaN, unread +//! by the contract) the kept slots give the oracle `card_sum` bit for bit — +//! fused, the fma the device compiles both kernels' `acc += w·down` to; a +//! place in `[n_card, HOST)` at its first value raises the layer's +//! `ExpertId` and NaNs every value of its token, the other tokens the +//! oracle's. //! Last, on the engine itself (`n_l` = 32): a NaN in a hybrid layer's input //! residual through `step_layer_hybrid` returns the card's fault at that //! layer — its norm's and its router's sites in the mask, printed in the @@ -114,6 +115,7 @@ fn main() -> std::process::ExitCode { #[cfg(feature = "gpu")] mod gate { + use bloomery_gpu::card_slots::{CardAcc, CardSlotKernels}; use bloomery_gpu::fused::FusedKernels; use bloomery_gpu::hybrid::{ Boundary, BoundaryShape, HOST, HostExperts, Hybrid, HybridConfig, Poison, PoisonKind, @@ -134,6 +136,7 @@ mod gate { use gguf::{GgmlType, Gguf, Split}; use model::arch::deepseek2::derived::Derived; use model::{Tensor2, Tensor2View}; + use runtime::combine::card_sum; use std::alloc::{GlobalAlloc, Layout, System}; use std::sync::atomic::{AtomicU64, Ordering}; @@ -1350,13 +1353,13 @@ mod gate { ok &= pass; } - // The card's routed combine (`weighted_sum_card`) over this - // boundary's places, nine slots a token (Bolt's count), three - // tokens: with every place a card slot it is `weighted_sum` bit for - // bit — the two kernels are the same expression, order and - // contraction; with HOST slots their down rows hold NaN (the - // contract leaves them unread) and the kept slots still give the - // host transcription of the rule bit for bit; a place in + // The card's routed combine (`ds41_ffn_card_acc_9`, the card slots' + // family at nine) over this boundary's places, nine slots a token + // (Bolt's count), three tokens: with every place a card slot it is + // `weighted_sum` bit for bit — the two kernels are the same + // expression, order and contraction; with HOST slots their down + // rows hold NaN (the contract leaves them unread) and the kept + // slots still give the oracle bit for bit; a place in // `[n_card, HOST)` names neither a card slot nor the host, so the // layer's ExpertId is raised and every value of its token is NaN // while the other tokens keep theirs. @@ -1376,18 +1379,17 @@ mod gate { let sel_all: Vec = (0..M * E) .map(|i| ((i / E + i % E) % R_CARD) as u32) .collect(); - // The rule itself, as the host reads it: ascending slots, the - // kept slots' `acc += w * down` from 0 — fused, because the - // device compiles the kernels' `acc += w * down` (both entries, - // `weighted_sum` included) to an fma. + // The rule itself, as the host reads it: the shared oracle + // `runtime::combine::card_sum`, ascending slots from zero — + // fused, because the device compiles the kernels' + // `acc += w * down` (both entries, `weighted_sum` included) to + // an fma. let sum_over = |down: &[f32], sel: &[u32], t: usize, d: usize| { - let mut acc = 0.0f32; - for e in 0..E { - if sel[t * E + e] < n_card { - acc = w[t * E + e].mul_add(down[(t * E + e) * rows + d], acc); - } - } - acc + card_sum::( + std::array::from_fn(|e| down[(t * E + e) * rows + d]), + std::array::from_fn(|e| w[t * E + e]), + std::array::from_fn(|e| sel[t * E + e] < n_card), + ) }; let transcription = |down: &[f32], sel: &[u32]| -> Vec { (0..M) @@ -1396,6 +1398,27 @@ mod gate { }; let down_dev = DeviceBuffer::from_host(stream, &down)?; let w_dev = DeviceBuffer::from_host(stream, &w)?; + let kernels = CardSlotKernels::load(gpu.context())?; + // The nine-slot card sum through the card slots' family, the + // buffers and extents the engine's card leg passes it. + let sum = + |down: &DeviceBuffer, sel: &DeviceBuffer, acc: &mut DeviceBuffer| { + kernels.enqueue_card_acc( + stream, + &CardAcc { + down, + w: &w_dev, + sel, + n: rows, + m: M, + n_card: R_CARD, + n_used: E, + w_pitch: E, + fault: gpu.layer_sink(R_LAYER)?, + }, + acc, + ) + }; // Every place a card slot: the card combine is the plain one, // bit for bit, and reruns identical. @@ -1405,46 +1428,24 @@ mod gate { let mut card = DeviceBuffer::::zeroed(stream, rows * M)?; gpu.elem() .enqueue_weighted_sum(stream, &down_dev, &w_dev, rows, E as u32, M, &mut full)?; - gpu.elem().enqueue_weighted_sum_card( - stream, - &down_dev, - &w_dev, - &sel_dev, - n_card, - rows, - E as u32, - M, - gpu.layer_sink(R_LAYER)?, - &mut card, - )?; + sum(&down_dev, &sel_dev, &mut card)?; stream.synchronize()?; let full_y = full.to_host_vec(stream)?; let card_y = card.to_host_vec(stream)?; - gpu.elem().enqueue_weighted_sum_card( - stream, - &down_dev, - &w_dev, - &sel_dev, - n_card, - rows, - E as u32, - M, - gpu.layer_sink(R_LAYER)?, - &mut card, - )?; + sum(&down_dev, &sel_dev, &mut card)?; stream.synchronize()?; let rerun = bits_equal(&card_y, &card.to_host_vec(stream)?); let same = bits_equal(&full_y, &card_y); let pass = same && rerun && gpu.take_fault()?.is_none(); println!( - "refusal weighted_sum_card every slot a card slot: == weighted_sum bits {same}, \ + "refusal card_acc_9 every slot a card slot: == weighted_sum bits {same}, \ rerun identical {rerun}, word clean {}", verdict(pass) ); ok &= pass; // HOST slots at chosen places, their down rows NaN: the kept - // slots' sum is the transcription bit for bit. + // slots' sum is `card_sum` bit for bit. gpu.clear_fault()?; let mut down_h = down.clone(); let mut sel_h = sel_all.clone(); @@ -1460,47 +1461,25 @@ mod gate { let want_h = transcription(&down_h, &sel_h); let down_h_dev = DeviceBuffer::from_host(stream, &down_h)?; let sel_h_dev = DeviceBuffer::from_host(stream, &sel_h)?; - gpu.elem().enqueue_weighted_sum_card( - stream, - &down_h_dev, - &w_dev, - &sel_h_dev, - n_card, - rows, - E as u32, - M, - gpu.layer_sink(R_LAYER)?, - &mut card, - )?; + sum(&down_h_dev, &sel_h_dev, &mut card)?; stream.synchronize()?; let host_y = card.to_host_vec(stream)?; let pass = bits_equal(&host_y, &want_h) && gpu.take_fault()?.is_none(); println!( - "refusal weighted_sum_card HOST slots skipped, their rows unread (NaN): == the \ - host transcription bits {}", + "refusal card_acc_9 HOST slots skipped, their rows unread (NaN): == card_sum \ + bits {}", verdict(pass) ); ok &= pass; - // A place in [n_card, HOST): the layer's ExpertId, that token - // every value NaN, the others the transcription. + // A place in [n_card, HOST), at its first value: the layer's + // ExpertId, that token every value NaN, the others the oracle. gpu.clear_fault()?; let mut sel_bad = sel_h.clone(); - sel_bad[E + 6] = n_card + 3; + sel_bad[E + 6] = n_card; let want_bad = transcription(&down_h, &sel_bad); let sel_bad_dev = DeviceBuffer::from_host(stream, &sel_bad)?; - gpu.elem().enqueue_weighted_sum_card( - stream, - &down_h_dev, - &w_dev, - &sel_bad_dev, - n_card, - rows, - E as u32, - M, - gpu.layer_sink(R_LAYER)?, - &mut card, - )?; + sum(&down_h_dev, &sel_bad_dev, &mut card)?; stream.synchronize()?; let bad_y = card.to_host_vec(stream)?; let token = |y: &[f32], t: usize| y[t * rows..(t + 1) * rows].to_vec(); @@ -1511,8 +1490,8 @@ mod gate { let word_ok = word == Some(Fault::at(layer, FaultSite::ExpertId)); let pass = bad_nan && others && word_ok; println!( - "refusal weighted_sum_card a place in [n_card, HOST): token NaN {bad_nan}, other \ - tokens the transcription {others}, word \"{}\" {}", + "refusal card_acc_9 a place in [n_card, HOST) at n_card: token NaN {bad_nan}, \ + other tokens the oracle {others}, word \"{}\" {}", word.map_or_else(|| "none".to_string(), |f| f.to_string()), verdict(pass) ); diff --git a/crates/gpu/src/card_slots.rs b/crates/gpu/src/card_slots.rs index 869c65b33..a7d42e8f5 100644 --- a/crates/gpu/src/card_slots.rs +++ b/crates/gpu/src/card_slots.rs @@ -42,6 +42,8 @@ const _: () = assert!(THREADS == 256); pub const SLOTS_6: usize = 6; /// Slots a token of the `_8` sum, whose contract spells it as a literal. pub const SLOTS_8: usize = 8; +/// Slots a token of the `_9` sum, whose contract spells it as a literal. +pub const SLOTS_9: usize = 9; /// Slots a token of the `_10` sum, whose contract spells it as a literal. pub const SLOTS_10: usize = 10; @@ -53,6 +55,8 @@ pub enum CardSlots { Six, /// Eight slots a token (GLM-5.3-Flash). Eight, + /// Nine slots a token (Bolt). + Nine, /// Ten slots a token, the router's weights at a pitch of eleven /// (Qwen3.8). Ten, @@ -65,6 +69,7 @@ impl CardSlots { match n_used { SLOTS_6 => Some(CardSlots::Six), SLOTS_8 => Some(CardSlots::Eight), + SLOTS_9 => Some(CardSlots::Nine), SLOTS_10 => Some(CardSlots::Ten), _ => None, } @@ -76,6 +81,7 @@ impl CardSlots { match self { CardSlots::Six => SLOTS_6, CardSlots::Eight => SLOTS_8, + CardSlots::Nine => SLOTS_9, CardSlots::Ten => SLOTS_10, } } @@ -86,6 +92,7 @@ impl CardSlots { match self { CardSlots::Six => "ds41_ffn_card_acc", CardSlots::Eight => "ds41_ffn_card_acc_8", + CardSlots::Nine => "ds41_ffn_card_acc_9", CardSlots::Ten => "ds41_ffn_card_acc_10", } } @@ -95,7 +102,7 @@ impl CardSlots { GpuError::Shape { what: "ds41_ffn_card_acc", detail: format!( - "{n_used} routed slots a token: the card sum's entries are built for 6, 8 and 10" + "{n_used} routed slots a token: the card sum's entries are built for 6, 8, 9 and 10" ), } } @@ -310,6 +317,48 @@ mod card_slots_kernels { // SAFETY: i < n·m <= acc.len(); thread i is acc[i]'s only writer. unsafe { *acc.get_unchecked_mut(i) = v }; } + + /// [`ds41_ffn_card_acc`] of nine slots a token ([`card_acc_at`]). + #[allow( + clippy::too_many_arguments, + reason = "kernel entry: the device ABI takes the arguments flat (rust-quality R8)" + )] + #[kernel] + #[launch_bounds(256)] + #[launch_contract( + domain = 1, + block = (256, 1, 1), + requires = ( + w_pitch >= 9, + down.len() >= 9 * n * m, + w.len() >= w_pitch * m, + sel.len() >= 9 * m, + acc.len() >= n * m + ) + )] + pub fn ds41_ffn_card_acc_9( + down: &[f32], + w: &[f32], + sel: &[u32], + n: u32, + m: u32, + n_card: u32, + w_pitch: u32, + fault: FaultSink, + mut acc: DisjointSlice, + ) { + let (n, m) = (n as usize, m as usize); + let i = thread::index_1d().get(); + if i >= n * m { + return; + } + // SAFETY: i < n·m, and the launch contract gives every length the + // helper's contract asks for at SLOTS_9 slots. + let v = + unsafe { card_acc_at::(down, w, sel, n, n_card, w_pitch as usize, fault, i) }; + // SAFETY: i < n·m <= acc.len(); thread i is acc[i]'s only writer. + unsafe { *acc.get_unchecked_mut(i) = v }; + } } /// The card slots' sum of value `i` (token `t = i / n`, value `d = i % n`) @@ -575,6 +624,12 @@ impl CardSlotKernels { stream, &prep, a.down, a.w, a.sel, n, m, n_card, w_pitch, a.fault, acc, )?; } + CardSlots::Nine => { + let prep = mo.prepare_ds41_ffn_card_acc_9(cfg)?; + mo.ds41_ffn_card_acc_9( + stream, &prep, a.down, a.w, a.sel, n, m, n_card, w_pitch, a.fault, acc, + )?; + } CardSlots::Ten => { let prep = mo.prepare_ds41_ffn_card_acc_10(cfg)?; mo.ds41_ffn_card_acc_10( @@ -605,7 +660,7 @@ mod tests { ) } - /// The sum at 6, 8 and 10 slots (the weights at the router's pitch of + /// The sum at 6, 8, 9 and 10 slots (the weights at the router's pitch of /// eleven at ten): a token whose slots are on the card or [`HOST`] is /// `runtime::combine::card_sum`'s and raises nothing; a place in /// `[n_card, HOST)` — the first past the card and the last before the @@ -631,7 +686,7 @@ mod tests { (state >> 40) as f32 / (1u64 << 24) as f32 - 0.5 }; let mut red = Vec::new(); - for n_used in [SLOTS_6, SLOTS_8, SLOTS_10] { + for n_used in [SLOTS_6, SLOTS_8, SLOTS_9, SLOTS_10] { let w_pitch = if n_used == SLOTS_10 { n_used + 1 } else { @@ -703,6 +758,7 @@ mod tests { let want = match n_used { SLOTS_6 => rule::(args.0, args.1, args.2), SLOTS_8 => rule::(args.0, args.1, args.2), + SLOTS_9 => rule::(args.0, args.1, args.2), _ => rule::(args.0, args.1, args.2), }; g.to_bits() != want.to_bits() @@ -730,7 +786,7 @@ mod tests { } // A card of no expert: every place is the host's, and the sum is // zeros with nothing raised. - for n_used in [SLOTS_6, SLOTS_8, SLOTS_10] { + for n_used in [SLOTS_6, SLOTS_8, SLOTS_9, SLOTS_10] { let sel = DeviceBuffer::from_host(stream, &vec![HOST; M * n_used]).expect("sel"); let z = DeviceBuffer::from_host(stream, &vec![f32::NAN; M * n_used * N]).expect("z"); let mut acc = DeviceBuffer::from_host(stream, &[1.0f32; N * M]).expect("acc"); diff --git a/crates/gpu/src/elem.rs b/crates/gpu/src/elem.rs index 75a6dc47e..5044669f1 100644 --- a/crates/gpu/src/elem.rs +++ b/crates/gpu/src/elem.rs @@ -12,9 +12,7 @@ //! `width * m` spans; `rope` on `[n_dims, n_vec, m]` (column `c = t*n_vec + //! v`, the layout of the reference dump's `q_rope`/`k_rope` views); //! `weighted_sum` on `[rows, n_exp, m]` down-projections with `[n_exp, m]` -//! weights, and `weighted_sum_card` over the card's share of the same (the -//! places `[n_exp, m]` beside the weights, the host's skipped); the embedding -//! table is Q3_K rows of 2048 values (880 bytes = 220 +//! weights; the embedding table is Q3_K rows of 2048 values (880 bytes = 220 //! u32 words each), Q3_K, Q4_K, Q5_K or Q6_K rows of whole super-blocks, or //! Q5_0 rows in the `pack_q5_0` layout. //! Extents are launch arguments, never buffer lengths: scratch buffers may @@ -545,46 +543,19 @@ pub(crate) unsafe fn weighted_expert_sum( n_exp: u32, t: usize, d: usize, -) -> f32 { - // SAFETY: every slot kept, so this fn's `# Safety` is the helper's. - unsafe { weighted_expert_sum_keep(down, w, rows, n_exp, t, d, |_| true) } -} - -/// [`weighted_expert_sum`] over the slots `keep` names: the same loop with -/// the refused slots' terms left out, their weight and down row unread. The -/// kept slots' expression, order and contraction are the plain sum's own, so -/// a caller that keeps every slot compiles to the plain sum's instructions; -/// the loop is this one helper so the two entries cannot drift apart. -/// -/// # Safety -/// -/// [`weighted_expert_sum`]'s contract, for the slots `keep` names; the -/// predicate's own reads are its caller's. -#[inline(always)] -pub(crate) unsafe fn weighted_expert_sum_keep( - down: &[f32], - w: &[f32], - rows: u32, - n_exp: u32, - t: usize, - d: usize, - mut keep: impl FnMut(usize) -> bool, ) -> f32 { let rows = rows as usize; let n_exp = n_exp as usize; let mut acc = 0.0f32; let mut e = 0usize; while e < n_exp { - if keep(e) { - // SAFETY: e < n_exp and t < m bound the weight index t*n_exp + e - // inside `w` by this fn's `# Safety`. - let wv = unsafe { *w.get_unchecked(t * n_exp + e) }; - // SAFETY: the same e < n_exp and t < m, with d < rows, bound the - // down index (t*n_exp + e)*rows + d inside `down` by this fn's - // `# Safety`. - let dv = unsafe { *down.get_unchecked((t * n_exp + e) * rows + d) }; - acc += wv * dv; - } + // SAFETY: e < n_exp and t < m bound the weight index t*n_exp + e + // inside `w` by this fn's `# Safety`. + let wv = unsafe { *w.get_unchecked(t * n_exp + e) }; + // SAFETY: the same e < n_exp and t < m, with d < rows, bound the down + // index (t*n_exp + e)*rows + d inside `down` by this fn's `# Safety`. + let dv = unsafe { *down.get_unchecked((t * n_exp + e) * rows + d) }; + acc += wv * dv; e += 1; } acc @@ -1250,72 +1221,6 @@ mod elem_kernels { } } - /// The routed-expert combine over the card's slots, one thread per - /// (token, output value): `weighted_sum`'s rule over the slots whose - /// place `sel[t*n_exp + e]` is below `n_card` — the same expression, - /// order and contraction, so with every slot on the card the two are bit - /// for bit equal. A place at [`HOST`] is the host's slot, skipped with - /// its down row and weight unread; a place in `[n_card, HOST)` names - /// neither a card slot nor the host: it raises [`FaultSite::ExpertId`] - /// on `fault` and every value of its token is NaN. - #[allow( - clippy::too_many_arguments, - reason = "kernel entry: the device ABI takes the arguments flat (rust-quality R8)" - )] - #[kernel] - #[launch_bounds(256)] - #[launch_contract( - domain = 1, - block = (256, 1, 1), - requires = ( - down.len() >= rows * n_exp * m, - w.len() >= n_exp * m, - sel.len() >= n_exp * m, - y.len() >= rows * m - ) - )] - pub fn weighted_sum_card( - down: &[f32], - w: &[f32], - sel: &[u32], - n_card: u32, - rows: u32, - n_exp: u32, - m: u32, - fault: FaultSink, - mut y: DisjointSlice, - ) { - let i = thread::index_1d().get(); - if i >= rows as usize * m as usize { - return; - } - let (t, d) = (i / rows as usize, i % rows as usize); - let mut faulted = false; - // SAFETY: i < rows·m (the guard above) gives t < m and d < rows; the - // launch contract gives down.len() >= rows·n_exp·m, w.len() >= - // n_exp·m and sel.len() >= n_exp·m. - let acc = unsafe { - weighted_expert_sum_keep(down, w, rows, n_exp, t, d, |e| { - // SAFETY: t < m and e < n_exp bound t·n_exp + e inside `sel` - // by the launch contract. - let place = unsafe { *sel.get_unchecked(t * n_exp as usize + e) }; - let keep = place < n_card; - if !keep && place != HOST { - faulted = true; - if d == 0 { - fault.raise(FaultSite::ExpertId); - } - } - keep - }) - }; - let v = if faulted { f32::NAN } else { acc }; - // SAFETY: i < rows*m <= y.len() by the launch contract. - unsafe { - *y.get_unchecked_mut(i) = v; - } - } - /// Widen `n` f16 bit patterns (one per word of `bits`, the low 16 bits) /// to f32 through [`half_to_f32`], the decode every weight scale in this /// package goes through. It exists so the gate can hold that decode @@ -2431,70 +2336,6 @@ impl ElemKernels { Ok(()) } - /// Enqueue the routed-expert combine over the card's slots: - /// `enqueue_weighted_sum`'s buffers and extents, with `sel` the places - /// (`n_exp * m` u32, `[n_exp, m]` beside `w`) and `n_card` the card's - /// slot count — a place below `n_card` is that card slot's, [`HOST`] is - /// the host's (skipped, its weight and down row unread), and a place in - /// `[n_card, HOST)` raises [`FaultSite::ExpertId`] on `fault` and NaNs - /// every value of its token. Asynchronous, allocation-free, capturable. - #[allow( - clippy::too_many_arguments, - reason = "the combine's two buffers, places, card count, extents, fault and output (rust-quality R8)" - )] - pub fn enqueue_weighted_sum_card( - &self, - stream: &CudaStream, - down: &DeviceBuffer, - w: &DeviceBuffer, - sel: &DeviceBuffer, - n_card: u32, - rows: usize, - n_exp: u32, - m: usize, - fault: FaultSink, - y: &mut DeviceBuffer, - ) -> Result<(), GpuError> { - if rows == 0 || n_exp == 0 || m == 0 { - return Err(GpuError::shape( - "enqueue_weighted_sum_card", - format!("need rows/n_exp/m >= 1, got {rows}/{n_exp}/{m}"), - )); - } - if down.len() < rows * n_exp as usize * m - || w.len() < n_exp as usize * m - || sel.len() < n_exp as usize * m - || y.len() < rows * m - { - return Err(GpuError::shape( - "enqueue_weighted_sum_card", - format!( - "down.len() {} (need {}), w.len() {} (need {}), sel.len() {} (need {}), \ - y.len() {} (need {})", - down.len(), - rows * n_exp as usize * m, - w.len(), - n_exp as usize * m, - sel.len(), - n_exp as usize * m, - y.len(), - rows * m - ), - )); - } - let what = "enqueue_weighted_sum_card"; - let grid = launch_u32(what, "grid", (rows * m).div_ceil(256))?; - let rows = launch_u32(what, "rows", rows)?; - let m = launch_u32(what, "m", m)?; - let prep = self - .module - .prepare_weighted_sum_card(LaunchConfig1D::new(grid, 256, 0))?; - self.module.weighted_sum_card( - stream, &prep, down, w, sel, n_card, rows, n_exp, m, fault, y, - )?; - Ok(()) - } - /// Enqueue the f16 widening of `bits` (the low 16 bits of each word) /// into `y`, through the same `cores::half_to_f32` the gemvs and the /// embedding dequant call. The gate's instrument, not a step op. diff --git a/tools/ref/ptx-shapes.tsv b/tools/ref/ptx-shapes.tsv index 433a310c8..4117c7356 100644 --- a/tools/ref/ptx-shapes.tsv +++ b/tools/ref/ptx-shapes.tsv @@ -46,6 +46,7 @@ generate_ds41 ds41_ffn_card_acc 0 0 generate_ds41 ds41_ffn_card_acc_10 0 0 generate_ds41 ds41_ffn_card_acc_8 0 0 generate_ds41 ds41_ffn_card_acc_8_tier 0 0 +generate_ds41 ds41_ffn_card_acc_9 0 0 generate_ds41 ds41_ffn_card_acc_tier 0 0 generate_ds41 ds41_ffn_handoff 0 0 generate_ds41 ds41_ffn_handoff_10 0 0 @@ -349,7 +350,6 @@ generate_ds41 swiglu_quant32_sel 0 0 generate_ds41 tier_fault_copy 0 0 generate_ds41 touch 0 0 generate_ds41 weighted_sum 0 0 -generate_ds41 weighted_sum_card 0 0 # PIN(2026-09-25): ds41_attn_seg_stage is the staged prefix segment pass (prefill only, never on the decode path); its ring ⧺ staging source keeps a few registers more than ds41_attn_seg, which already sits at the 128-register cap of a 512-thread block, and ptxas spills 8 bytes. ds41_attn_seg_sel_stage fits in 112 with none. # gate_e2e (--features gpu) @@ -371,6 +371,7 @@ gate_e2e down_add_q5_1 0 0 gate_e2e ds41_ffn_card_acc 0 0 gate_e2e ds41_ffn_card_acc_10 0 0 gate_e2e ds41_ffn_card_acc_8 0 0 +gate_e2e ds41_ffn_card_acc_9 0 0 gate_e2e ds41_ffn_handoff 0 0 gate_e2e ds41_ffn_handoff_10 0 0 gate_e2e ds41_ffn_handoff_10_cols 0 0 @@ -622,7 +623,6 @@ gate_e2e swiglu_quant32_sel 0 0 gate_e2e tier_fault_copy 0 0 gate_e2e touch 0 0 gate_e2e weighted_sum 0 0 -gate_e2e weighted_sum_card 0 0 # gate_qwen3vl_tower (--features vision), the `vis_` entries of bloomery-gpu-vision only: V4.1's six, the Qwen3-VL tower's five, the glm5v tower's three, and MiniCPM's two gate_qwen3vl_tower vis_attn 0 0 diff --git a/tools/unsafe-ratchet.txt b/tools/unsafe-ratchet.txt index a7e86bfd4..a4043a147 100644 --- a/tools/unsafe-ratchet.txt +++ b/tools/unsafe-ratchet.txt @@ -273,7 +273,11 @@ gguf 9 2 0 # (bolt-b1-handoff9), ds41_ffn_handoff_9's call into handoff_at and weighted_sum_card's reads and write, and the # split of weighted_expert_sum's body into weighted_expert_sum_keep (the plain sum's call into it), each block with # its SAFETY line; the fn is weighted_expert_sum_keep itself. -gpu 1874 204 10 +# PIN(2026-10-17): gpu 1874 -> 1872 blocks, 204 -> 203 fns, the nine-slot card combine folded into +# the card slots' family (bolt-b1 r2): weighted_sum_card and the weighted_expert_sum_keep split +# deleted (the plain sum's body back), ds41_ffn_card_acc_9's call into card_acc_at and its write +# added, each block with its SAFETY line; the fn was the deleted keep. +gpu 1872 203 10 # PIN(2026-10-01): gpu-deepseek41 459 -> 464 blocks, 51 -> 52 fns, a new kernel entry, not old code relaxed: # ds41_ffn_card_acc_8_tier (the tier form of the card sum at 8 slots, chain/ffn/batch.rs) adds its two blocks and # card_acc_tier_at (the one unsafe fn) with its three, each with its SAFETY line. From 570ddb7840a3f409bf714d67e2b8490e19afd850 Mon Sep 17 00:00:00 2001 From: midagedev Date: Sat, 17 Oct 2026 06:39:20 +0900 Subject: [PATCH 3/3] gate: the prefill card sum stops refusing nine slots (review follow-up) The nine-slot entry is built, so the clause that refused 9 through CardSlots now refuses only 5 and 7, with the dated reason; the nine-slot entry itself is checked by gate_hybrid combine clauses and gate-gpu-lib places test. The gather clause keeps refusing 9: its family has no nine-slot entry. Co-Authored-By: Claude Code --- crates/gpu-gates/src/bin/gate_deepseek41_prefill.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/gpu-gates/src/bin/gate_deepseek41_prefill.rs b/crates/gpu-gates/src/bin/gate_deepseek41_prefill.rs index 9bf44dbc7..96a8506e7 100644 --- a/crates/gpu-gates/src/bin/gate_deepseek41_prefill.rs +++ b/crates/gpu-gates/src/bin/gate_deepseek41_prefill.rs @@ -1615,7 +1615,10 @@ mod gate { verdict(pass) ); } - for n_used in [5, 7, 9] { + // PIN(2026-10-16): the card sum stops refusing 9 (`CardSlots::Nine`); + // the nine-slot entry is checked by gate_hybrid's combine clauses and + // gate-gpu-lib's places test. + for n_used in [5, 7] { let z = DeviceBuffer::from_host(stream, &vec![0.0f32; SLOTS8 * N * M])?; let s = DeviceBuffer::from_host(stream, &[HOST; SLOTS8 * M])?; let mut acc = DeviceBuffer::from_host(stream, &[SENT; N * M])?;