diff --git a/.cargo/config-v4.toml b/.cargo/config-v4.toml new file mode 100644 index 00000000..c882d315 --- /dev/null +++ b/.cargo/config-v4.toml @@ -0,0 +1,22 @@ +[build] +# Plain AVX-512 baseline — `x86-64-v4` (F + BW + CD + DQ + VL), nothing above +# it. Use with: +# cargo --config .cargo/config-v4.toml check --lib +# cargo --config .cargo/config-v4.toml test --lib --no-run +# +# This is the deterministic AVX-512 COMPILE contract the SIMD realization +# matrix builds against: `simd.rs` keys its arm off `target_feature = "avx512f"`, +# so this selects `simd_avx512` and emits `vpternlogq` etc. regardless of the +# build host's silicon — code generation and execution are separate, and a +# GitHub runner does not need AVX-512 to prove LLVM emits it. Do NOT run the +# resulting binary on a host without AVX-512 (SIGILL); use the v3 or native +# configs for execution, or qemu for semantic parity. +# +# `config-avx512.toml` is the stricter Sapphire Rapids EXECUTION config (VNNI, +# BF16, FP16, AMX…); it SIGILLs on any earlier AVX-512 silicon, so it is never +# the CI compile oracle. +# `-Dwarnings` rides here because CI must `env -u RUSTFLAGS` to let this file +# apply at all (a RUSTFLAGS env replaces every config rustflags entry), and +# the repo rule is warnings-clean on every arm anyway. +[target.'cfg(target_arch = "x86_64")'] +rustflags = ["-Ctarget-cpu=x86-64-v4", "-Dwarnings"] diff --git a/.claude/AMX_GOTCHAS.md b/.claude/AMX_GOTCHAS.md index 1776104e..11273cd9 100644 --- a/.claude/AMX_GOTCHAS.md +++ b/.claude/AMX_GOTCHAS.md @@ -267,6 +267,25 @@ in production under load, AVX-512 siblings unaffected. --- +## Gotcha 15: the operand "mirror" was a misread of the byte table — use mnemonics + +`src/hpc/amx_ops.rs` (2026-09-14) assembles every AMX mnemonic on stable +1.98.1 with `const` tile operands. Intel order is `tdpbusd tmmD, tmmS1, tmmS2` += `D += S1·S2`, S1 = ModRM.rm (plain M×K), S2 = VEX.vvvv (VNNI K×N). The +validated `C4 E2 71 5E C2` is `tdpbusd tmm0, tmm2, tmm1`, i.e. the kernel's +"A in tmm2, B in tmm1" placement is the plain SDM semantics, not a mirror. +Gotcha 12's *placement* stays correct; its *explanation* is superseded. +Aliased tile operands are now a compile error (`const` assert), so the SIGILL +of Gotcha 11 cannot be written THROUGH `amx_ops` (the `.byte` path in +`amx_matmul` is untouched and still lets a caller alias tiles). Encoding +tests read each wrapper's bytes out of the test binary's ELF symtab and pin +at least one op of every tier that has a register-only or masked encoding: +the four GEMM-tier sequences to the EMR-validated table, everything else to +LLVM 22.1.8's own emission (a drift guard). TF32 is hand-encoded raw bytes +(nightly's LLVM 23 dropped the mnemonic) pinned to the bytes 22.1.8 once +produced. The AVX512 row ops are pinned only on `avx512f` builds. NO tier +beyond TILE/INT8/BF16 has executed on any host here. + ## Hardware tiers ``` diff --git a/.claude/blackboard.md b/.claude/blackboard.md index ebe3d534..45013786 100644 --- a/.claude/blackboard.md +++ b/.claude/blackboard.md @@ -1,3 +1,310 @@ +## 2026-09-14 — AVX2 arm of the mask family MEASURED, not rewritten: 6 of 10 shapes were already packed, 4 earned intrinsic realizations + +**The pre-compaction plan was wrong, and the instrument said so before code +was written.** The five-flavour audit (entry below) scheduled a rewrite of +`simd_avx2.rs`'s `U64x8`/`I32x16` from `avx2_int_type!` array polyfills to +native `[__m256i; 2]` types. Before doing it I added the mask family to the +codegen oracle as Group F (`.claude/knowledge/simd-codegen-oracle/probes.rs`, +ten `#[inline(never)]` probes calling the SHIPPED library methods, each with +a runtime self-check against the bit-serial definition) and ran it on the +untouched polyfill at `-Ctarget-cpu=x86-64-v3`: + +| shape | packed / scalar-lane-arith | +|---|---| +| `ternlog::` / `::<0xCA>` u64x8 | 18 / 0 | +| `ternlog::` u32x16 | 8 / 0 | +| `andnot` u64x8 | 6 / 0 | +| `popcnt` u64x8 | 21 / 0 (vpshufb nibble LUT, not 8× popcntq) | +| `xor_popcount` u64x8 | 25 / 0 | +| **`rotate_left`** u64x8 | **0 / 8** `rolq` | +| **`reduce_max`** i32x16 | **0 / 17** `cmpl` | +| **`gt_bitmask`** i32x16 | **23 / 3** MIXED — lanes 0, 13–15 peeled to scalar | +| **`cmpge_zero_mask`** i32x16 | **17 / 11** MIXED — same peel | + +The whole bit-logic half — the generated Shannon ladders, andnot, popcount +— was packed from scalar source, exactly the oracle README's standing +finding ("a recent PR hand-wrote ~700 lines of intrinsics to fix a gap that +did not exist"). A `[__m256i; 2]` rewrite would have re-implemented six +already-packed shapes and broken every `.0[i]` site in the file's seven +`U64x8`/`I32x16`/`U32x16` impl blocks for nothing. + +**What shipped instead (backend-local, narrow `unsafe`, no `#[target_feature]`):** +`U64x8::rotate_left/right` → `vpsllq`+`vpsrlq`+`vpor` per 256-bit half +(uniform xmm count; 10 packed / 2 scalar — the 2 are `n % 64` / `64 - n` +count setup, not lane data); `I32x16::reduce_min/max` → `vpminsd`/`vpmaxsd` +tree 16→8→4→2→1 (8 / 0); `I32x16::gt_bitmask` → `vpcmpgtd` + movemask per +half (9 / 0); `I32x16::cmpge_zero_mask` → complemented sign-bit movemask +(10 / 0). SAFETY precondition on every block: this file is the x86-64-v3 +backend, `.cargo/config.toml` pins the target-cpu for every x86_64 build +that selects the arm — the footing the native `U16x16` already stood on. +Oracle re-run: ALL PROBES MATCH; Group F rows now carry `expect = +"vectorized"` with 60 % floors and both runs' numbers in the notes. + +**Two stale claims corrected in the same pass.** `I32x16::gt_bitmask`'s doc +comment said the oracle had measured a clean packed lowering for "exactly +this form" — it had not been probed; measured, it was the mixed peel above. +And `simd.rs`'s AVX2-arm comment claimed `simd_avx2.rs` carries +per-function `#[target_feature(enable = "avx,avx2,fma")]` — grep finds +zero, and by the operator's standing rule there must be none (one backend +file, one compile-time target). Both rewritten to what is true. + +**Instrument findings, recorded not explained.** (a) `scripts/neon-asm-rung3.sh` +misreported `check_ternlog_all_tables` as scalarised on its first real run +because it counted LLVM's `.LBB*` basic-block labels as symbol boundaries — +684 vector ops fragmented into hundreds of 4-op stubs. Fixed (function +symbols only) and the scalar reference oracles excluded from the gate: rung +3 PASS at 794 vector / 33 scalar. (b) The oracle's untouched hand-written +`shiftor_rot_u64x8` probe flipped 0→10 packed in the same run that made the +library rotate an intrinsic, while its two siblings stayed scalar; mechanism +not established, noted on its baseline row as a finding about the instrument. + +**Gates on the final tree:** v3 clippy `-D warnings` + full lib 2292/2292; +v4 clippy + 114 masking/simd tests; aarch64 check + rung 3 PASS; WASM parity +OK; oracle ALL MATCH. New falsifier +`i32x16_compare_bitmasks_and_reductions_at_lane_extremes` places the signed +extremes at lanes 0/7/8/15 and walks MIN/MAX through every lane, so a +wrong half order or a lane-dropping tree cannot pass it. + +## 2026-09-13 (2) — FIVE-FLAVOUR AUDIT of the masking substrate: U64x8/I32x16 were SCALAR on NEON, WASM and (as array polyfills) AVX2 + +**Operator correction, verbatim in substance:** *re-read the dispatch +architecture before using `safe_intrinsic_probe` to establish policy.* There +are five execution flavours — (1) x86-64-v3 default/CI → `simd_avx2`; +(2) AVX-512/v4 → `simd_avx512`; (3) `target-cpu=native` → backend from the +build host's CPUID; (4) `nightly-simd` → `simd_nightly`/`core::simd`; +(5) `--features runtime-dispatch` → one LazyLock capability detection, then the +selected kernel. **`#[target_feature]` propagation is NOT the architecture.** +For every compile-time flavour the selected backend file IS the capability +proof and raw intrinsics stay at a narrow backend-local `unsafe` boundary; for +flavour 5 the LazyLock branch is the proof; nightly inherits no ISA contract. +**Never route a mask primitive through Scalar because rustc wants `unsafe` at +an intrinsic.** (Measured: PR #306 adds no `#[target_feature]` outside the +probe's two demonstration arms; the worker briefs forbid it verbatim.) + +**The audit's finding, the one that mattered:** the mask family is built on +`U64x8` (all bulk algebra + ternlog) and `I32x16` (the signed-compare family), +and `simd.rs` resolved BOTH to the **scalar** backend on aarch64 +(`:390-393`) and wasm32 (`:408-414`), while the v3 arm's are `avx2_int_type!` +array polyfills. Only the `U32x16` paths (`eq_u32`/`ne_u32`/`ternary_match_u32`) +reached NEON/v128. So the "424 NEON vector ops" rung-3 measurement was on the +one lane type the family barely uses, and every `mask_and`/`mask_ternlog`/ +`gt_i32_to_mask` ran scalar loops on three of five flavours. The +`agnostic-surface-cpu-matrix.md` rows claiming NEON `4×uint64x2_t` / +`4×int32x4_t` were wrong (the dispatch-architecture matrix's ❌ was right). + +**Fix (PR1 scope — "backend-local polyfill completion"):** native `U64x8` +(`[uint64x2_t;4]` / `[v128;4]` / `[__m256i;2]`) and `I32x16` (`[int32x4_t;4]` / +`[v128;4]` / `[__m256i;2]`) in `simd_neon.rs`, `simd_wasm.rs`, `simd_avx2.rs` +with the FULL scalar `impl_int_type!` surface (so nothing that compiled +against the scalar re-export breaks — census: no consumer constructs these +types or reads `.0`); generator arms for their `ternlog`; `simd.rs` re-export +flip; harness arms (`check_u64x8_algebra`, `check_i32x16_compare`) on NEON and +WASM; `scripts/neon-asm-rung3.sh` — the rung-3 count made symmetric +(same mnemonic set on v-regs and w/x-regs, incl. `orn`/`mvn`), attributed per +symbol, with a gate that every `ternlog` symbol is vector-dominant. + +**Recorded limits, not fixed here:** flavour 4 — the mask family does not +compile under `nightly-simd` (`I32x16` has `cmpgt_mask`, not `gt_bitmask`; +`U64x8` has no `andnot`; `ternlog` is the 36-op minterm) — PRE-EXISTING (same +calls lived in `simd_int_ops` before #306; CI's nightly job is skipped). +> ⊘ SUPERSEDED within #306 (e730109 "nightly realization complete"): the +> nightly arm now carries `gt_bitmask`, `andnot`, `ternlog` and the W1a +> types; the parity program runs on it (`masking-parity.sh nightly`) and +> the matrix's nightly row exercises it. The limit above is history. +Flavour 5 — no mask trampolines in `simd_runtime`; a release binary runs the +v3-compiled mask kernels on every host. Both are separate decisions. + +**Council corrections folded in (C2, overclaim audit):** "≤ 7 ops" → 7/8 per +vocabulary, now asserted by the generator; "113/113" → the filter is named; +the x86 256-table sweep gained a `U32x16` twin; "public surface unchanged" → +facade-preserved, module paths removed; `#![forbid(unsafe_code)]` is now +declared on `simd_masking_ops.rs` rather than claimed; the wasm harness +comment no longer claims the NEON equality it does not run; `mask_any` is +documented tail-blind (its pair `mask_all` takes `n_rows`); the generator gained +`--check` (regenerate-and-diff; a hand-edited body is now detectable). + +## 2026-09-13 — `simd_masking_ops.rs` + generated backend-local `ternlog` bodies + the DuckDB-vector-execution primitive set (PR1 of the mask-RISC arc) + +**Three-layer contract, operator-ruled this session — the architecture law +this entry exists to make durable:** + +```text +consumers (lance-graph-mask-risc, lgj-abi kernels, planner) + │ semantic ops only: TERNLOG, AND, XOR, COUNT, eq→mask … + ▼ +simd_masking_ops.rs slice/chunk/tail ergonomics, *_assign forms, + │ mask composition, masked reductions — NEVER an ISA + ▼ +simd.rs architecture-agnostic lane types, compile-time selected + ▼ +simd_{avx512,avx2,neon,wasm,scalar}.rs each owns its realization, as a PEER +``` + +- **POLYFILL LAW.** ndarray is the ISA membrane. Every public mask/SIMD + primitive a consumer uses has compile-time implementations for AVX-512, + AVX2, NEON, WASM SIMD and scalar. **Scalar is a peer backend, not a + fallback.** No runtime ISA dispatch, no fallback chains. Hardware-specific + optimisation — including truth-table specialisation of `ternlog` — lives + entirely inside the corresponding backend file. Consumers never branch on + ISA; `TERNLOG` stays semantic above the backends. +- **BACKEND LAW.** No shared generic/polyfill implementation body that the + backends delegate into. Shared *tests* and shared *generated truth-table + logic* are fine; a shared *runtime* body is not. The route to remove + repeated source is code generation emitting backend-LOCAL bodies. + +**What landed:** + +> **AMX fill (2026-09-14):** `src/hpc/amx_ops.rs` — the surviving +> `X86InstrAMX.td` surface (amx-transpose is absent by design) as mnemonics +> with `const` tile operands (INT8×4, BF16, FP16, COMPLEX×2, FP8×4, MOVRS×2, +> AVX512 row ops ×6 under `avx512f` cfg, STTILECFG/TILELOADDT1, all 8 tiles) +> plus TF32 as hand-encoded raw bytes (nightly's LLVM 23 rejects the +> mnemonic; b80fd83). `amx_features()` per LLVM `Host.cpp` bits; +> `amx_report()` prints the tiers. Encoding falsifiers read each wrapper's +> bytes out of the test binary's ELF symtab on any x86 host: the four +> GEMM-tier sequences are pinned to the EMR-VALIDATED table, every other +> pinned op is pinned to LLVM's own emission (a drift guard, not silicon +> validation). The "mirrored operand convention" turned out to be a misread +> of that table (Gotcha 15). Extended tiers are assembler-verified only. + +> **CI finding (2026-09-14, e730109 red on `tier4-avx512-check`):** `ci.yaml`'s +> workflow-global `RUSTFLAGS: "-D warnings"` REPLACES every `.cargo/config*` +> rustflags entry (cargo precedence: RUSTFLAGS > target. > target. +> > build). Consequences, both measured: (a) the v4 job never had a +> target-cpu, with the env-var recipe (which loses to the joined cfg `v3` +> locally) or with `--config` (erased by the global env in CI) — the new +> vpternlog assertion caught it at 0; (b) EVERY x86 job in CI builds at the +> x86-64 baseline, without the v3 pin and without the dalek/poly1305 `--cfg`s +> that `.cargo/config.toml` exists to apply. (a) is fixed in this PR +> (`env -u RUSTFLAGS` + `--config .cargo/config-v4.toml`, `-Dwarnings` moved +> into that file). (b) is pre-existing and out of this PR's concern: the fix +> is a triple-scoped `CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUSTFLAGS` for the +> x86 jobs plus per-target flags for nostd/wasm/aarch64, its own PR. + +1. **`src/simd_masking_ops.rs`** — the mask family moved out of + `simd_int_ops.rs` wholesale (predicates→mask, mask algebra, ternlog, + masked reductions, care-masked register match, blend) with its tests. + `simd_int_ops.rs` is integer arithmetic/conversion again. Every moved + `pub fn` (31/31) still re-exports through `ndarray::simd`, AND the 13 + mask functions that were public on master as `ndarray::simd_int_ops::` + (`simd_int_ops` is `pub mod`, so those were public paths — the first + draft dropped them and called the surface "unchanged", C2; CodeRabbit + round 2 caught the downstream break) are re-exported from `simd_int_ops` + as a compatibility surface, verified complete by diffing master's + `pub fn` list against HEAD's `pub fn` + `pub use` set (0 missing). The + canonical path is the facade; `lance-graph-planner` + `examples/dcr_w0_replay_budget.rs` moves to it in the lance-graph PR. +2. **`tools/gen_ternlog_bodies.py`** — Shannon-lowers each 8-bit table into + two 2-input tables (`f = (!c & T0) | (c & T1)`), ≤ 7 ops (the naive + minterm form was up to 36), self-checks all 256 tables in Python, and + PRINTS each backend's body in that backend's own vocabulary between + `GEN-TERNLOG` markers (worst case **7 ops** where the vocabulary has a + native and-not — NEON `vbic`, WASM `v128.andnot` — and **8** where and-not + is spelled `x & !y`, the avx2/scalar operator vocabularies; the generator + ASSERTS these bounds on the emitted text, `count_ops`; the earlier "≤ 7 for + any table" was wrong for two of four backends — C2 council finding): operator traits on the array lanes (avx2, scalar), + per-`u32`-lane for NEON (`#[cfg(target_arch = "aarch64")]`-gated helper), + `v128_*` intrinsics for WASM (helper inside the cfg-gated `wasm32_simd` + module). AVX-512 keeps `_mm512_ternarylogic_epi64` untouched. The generic + `simd_ternlog_lower.rs` that a first cut shared across four backends was + DELETED — it violated the Backend Law and also blew the debug stack + (`#[inline(always)]` × 256 tables in one test frame). + Two generator traps recorded: a `const` item cannot read the enclosing + fn's `IMM` (E0401) — the tables are `let`-bound and fold identically after + monomorphisation; and the helper must be placed INSIDE the cfg-gated + module or every host compiles it and fails to resolve the intrinsics. +3. **New primitives** (all through `ndarray::simd`): `lt/ge/le/ne/eq_i32_to_mask`, + `ne_u32_to_mask`, `mask_not{,_assign}` (tail re-cleared against `n_rows`), + `mask_xor{,_assign}` (its own primitive, lane `^` — NOT `ternlog::`, + whose AVX2 minterm cost is pointless for a native op), `mask_any`, + `mask_all`, `ternary_match_{u32,u64,strided}_to_mask` (care-masked + register match via `ternlog::` + zero test — the TCAM shape of a + V3 12-byte facet), `masked_min/max_i32`, `blend_i32`; immediates + `XOR_AND = 0x28`, `AND2_OR = 0xEA`. Ordered compares derive from `gt` + by complement, so they are exact at `i32::MIN`/`MAX` (threshold shifting + underflows). + +**Acceptance matrix, measured (not asserted):** for every IMM in 0..=255, +bit-serial reference == the compiled realisation — +x86 arms, `cargo test --lib -- simd_masking_ops::tests simd_int_ops::tests +simd::tests` (a FILTER — the lib suite is ~3,100 tests; 113 is the selected +set): v3 113/113, v4 (separate target dir) 113/113. On x86 the 256-table +sweep now runs on BOTH lane types (`U64x8` and, since the C2 finding, a +distinct `U32x16` sweep — the v3 arm carries two separate generated +ladders and v4 two different intrinsics, so one sweep proved nothing about +the other); **WASM: run for real under node** via +`scripts/wasm-parity.sh`, whose harness gained `check_ternlog_all_tables` +(256 tables × `U32x16` native v128 body × `U64x8` scalar body, two operand +triples each) — rc=0; **NEON: rungs 1 and 3 of the AArch64 ladder measured on this host** — +`cargo check --target aarch64-unknown-linux-gnu` of lib+tests and the +harness (all 256 monomorphisations) compile; the cross-compiled harness +assembly selects **424 NEON vector logical ops** (`and/orr/eor/bic/orn +v.16b` — LLVM fuses `orr(mvn)` into `orn`) against 41 scalar ops left in +harness scaffolding. The FIRST generated NEON body — a per-`u32`-lane loop +through `to_array()`/`from_array()` — SCALARIZED: 536 scalar vs 4 vector +ops. Same truth tables, same tests, rung 3 red. The body is now emitted +per 128-bit quad in the backend's own intrinsic vocabulary +(`vandq/vorrq/veorq/vbicq/vmvnq_u32` on `uint32x4_t`). Rung 2 (run under +qemu) is CI's `neon_simd` job — no cross linker / qemu on this host, stated +not assumed; rung 5 (Apple/AArch64 hardware) is a later performance gate, +never a blocker for authoring the backend. Both harness arms use the SAME check body — shared tests are +allowed, shared implementation is not. + +**AArch64 acceptance ladder (operator, 2026-09-13)** — the LLVM/Clang +intrinsic corpus is the remote instruction catalogue for a backend the +author cannot run: (1) cross-target compile succeeds; (2) parity harness +compiles/runs under an emulator where sensible; (3) generated LLVM IR / +assembly contains the expected NEON operations and no unexpected +scalarisation; (4) truth-table / reference parity is exhaustive where +possible; (5) real hardware benchmarking is a LATER performance gate. +Three different proofs, kept sharp: LLVM says what lowering is available, +cross-compiled assembly says what LLVM actually selected, hardware says +whether the selection is fast. + +**Two standing rules (operator, 2026-09-13), recorded where the next +backend author will look:** + +- **97 % safe. `unsafe` only for byte-code asm (AMX-class inline asm).** + Operator, on the intrinsic question: *"98 % of intrinsics are available in + safe mode by rust 1.98.1 — if not, document where and why."* Measured on + the pinned 1.98.1 with `tools/safe_intrinsic_probe` (re-run after every + toolchain bump; the answer is a toolchain property): + + | arch / call shape | 1.98.1 | + |---|---| + | aarch64: plain fn → `vandq_u32` | **E0133** — caller must carry `#[target_feature(enable = "neon")]`; build-config `neon` "does not remove the requirement" | + | aarch64: `#[target_feature(neon)]` fn → `vandq_u32` | OK (safe, no `unsafe`) | + | aarch64: plain fn → that safe annotated fn | **E0133** — the requirement propagates up the chain | + | x86_64: plain fn → `_mm_and_si128` (sse2, baseline) | **E0133** | + | x86_64: plain fn → `_mm256_and_si256`, even with `-Ctarget-cpu=x86-64-v3` | **E0133** | + | x86_64: plain fn → `_mm512_ternarylogic_epi64`, even with `-Ctarget-cpu=x86-64-v4` | **E0133** | + | wasm32: plain fn → `v128_and`, with or without `+simd128` | **OK** | + + So the intrinsic *functions* are safe, but rustc only accepts a per-fn + `#[target_feature]` as evidence — and in this repo that evidence is + **illogical to state** (operator ruling): every `simd_{arch}.rs` is compiled + for exactly one target CPU, selected by `cfg` at compile time, so the + feature is already a property of the file. Annotating each fn would be a + second, redundant declaration of the same fact, and it would propagate to + every safe caller up to the pub boundary. rustc simply does not read the + `cfg` as proof. Consequence: one expression-narrow `unsafe` at the + intrinsic boundary per backend method, with a SAFETY line (the generated + NEON body); the generated WASM body carries none; `simd_masking_ops.rs` + and every consumer above it stay `forbid(unsafe_code)`. Follow-up, not + this PR: the 20 pre-existing `unsafe` blocks in `simd_wasm.rs` are + removable under this finding. +- **Conversions are bit-exact; rounding happens at most once.** F32 → + BF16x16 rounds exactly once, through a fused `add_mul` — never a separate + multiply then add, never a convert-then-convert. Mask primitives carry no + floats, so this PR is unaffected; the rule binds the BF16 lanes and every + future reduction that touches them. + +**Loose ends:** `simd_masking_ops` still has only slice-level compositions +that consumers already needed; the ergonomic fused forms the mask-RISC +executor will want (`masked_count_where_eq`, chunked survivor-word skip +helpers) land with that consumer, backend-first. A `stride_bytes == 8` twin +of the contiguous `eq_u32_strided` fast path is still unbuilt (no caller). + ## 2026-09-05 — D-GTM-0l MEASURED (prefix-tract coverage, R2IL 6502 ore) The probe I flagged as decisive ran. `examples/prefix_tract_coverage_probe.rs` diff --git a/.claude/knowledge/agnostic-surface-cpu-matrix.md b/.claude/knowledge/agnostic-surface-cpu-matrix.md index d3050201..cf5bbdcd 100644 --- a/.claude/knowledge/agnostic-surface-cpu-matrix.md +++ b/.claude/knowledge/agnostic-surface-cpu-matrix.md @@ -85,15 +85,17 @@ two `__m256i` halves; "4×NEON" means four 128-bit NEON registers (e.g. | `I16x16` | `__m256i` | ← | ← | ← | ← | ← | ← | ← | `__m256i` | `__m256i` | 2×`int16x8_t` | ← | ← | `[i16;16]` | | `U16x32` | `__m512i` | ← | ← | ← | ← | ← | ← | ← | 2×`__m256i`⏳| 2×`__m256i`⏳| 4×`uint16x8_t` | ← | ← | `[u16;32]` | | `U16x16` | `__m256i` | ← | ← | ← | ← | ← | ← | ← | `__m256i` | `__m256i` | 2×`uint16x8_t` | ← | ← | `[u16;16]` | -| `I32x16` | `__m512i` | ← | ← | ← | ← | ← | ← | ← | 2×`__m256i`| 2×`__m256i`| 4×`int32x4_t` | ← | ← | `[i32;16]` | +| `I32x16` | `__m512i` | ← | ← | ← | ← | ← | ← | ← | 2×`__m256i`†| 2×`__m256i`†| 4×`int32x4_t`† | ← | ← | `[i32;16]` | | `I32x8` | `__m256i` | ← | ← | ← | ← | ← | ← | ← | `__m256i` | `__m256i` | 2×`int32x4_t` | ← | ← | `[i32;8]` | | `U32x16` | `__m512i` | ← | ← | ← | ← | ← | ← | ← | 2×`__m256i`⏳| 2×`__m256i`⏳| 4×`uint32x4_t` | ← | ← | `[u32;16]` | | `U32x8` | `__m256i` | ← | ← | ← | ← | ← | ← | ← | `__m256i`⏳ | `__m256i`⏳ | 2×`uint32x4_t` | ← | ← | `[u32;8]` | | `I64x8` | `__m512i` | ← | ← | ← | ← | ← | ← | ← | 2×`__m256i`| 2×`__m256i`| 4×`int64x2_t` | ← | ← | `[i64;8]` | | `I64x4` | `__m256i` | ← | ← | ← | ← | ← | ← | ← | `__m256i` | `__m256i` | 2×`int64x2_t` | ← | ← | `[i64;4]` | -| `U64x8` | `__m512i` | ← | ← | ← | ← | ← | ← | ← | 2×`__m256i`| 2×`__m256i`| 4×`uint64x2_t` | ← | ← | `[u64;8]` | +| `U64x8` | `__m512i` | ← | ← | ← | ← | ← | ← | ← | 2×`__m256i`†| 2×`__m256i`†| 4×`uint64x2_t`† | ← | ← | `[u64;8]` | | `U64x4` | `__m256i` | ← | ← | ← | ← | ← | ← | ← | `__m256i` | `__m256i` | 2×`uint64x2_t` | ← | ← | `[u64;4]` | +† = native since 2026-09-13 (PR #306 five-flavour audit; the AVX2 column's "2×`__m256i`" is the storage SHAPE the codegen lowers to — the type stays the `#[repr(align(64))]` `[T; N]` array, measured packed for the bit-logic half and given two-half intrinsic bodies for rotate / reduce / compare-bitmask on 2026-09-14). Until then these two rows were WRONG: on aarch64 and wasm32 `simd.rs` re-exported the SCALAR `U64x8`/`I32x16`, and on the v3 arm they were `avx2_int_type!` array polyfills — the mask family (`simd_masking_ops`) rides exactly these two types, so it ran scalar on three of five flavours. + ⏳ = TD-T22 polyfill audit — the 256-bit `U16x16/U16x32/U32x8/U32x16` inner ops may currently use scalar storage under `#[target_feature]` rather than real `__m256i` intrinsics. Needs verification (see § J integration plan). diff --git a/.claude/knowledge/amx-enablement-and-kernel.md b/.claude/knowledge/amx-enablement-and-kernel.md index d2df5ec1..73057653 100644 --- a/.claude/knowledge/amx-enablement-and-kernel.md +++ b/.claude/knowledge/amx-enablement-and-kernel.md @@ -169,15 +169,68 @@ which is bug #2. For the 16×16 int8/bf16 tile, all three tiles are 16 rows × --- +## 5b. The mnemonic surface — `src/hpc/amx_ops.rs` (2026-09-14, LLVM 22.1.8) + +The `.byte` tables above were forced by 1.94. Measured on **1.98.1 (LLVM +22.1.8)**: the integrated assembler accepts EVERY AMX mnemonic inside `asm!` +with no target feature, and `asm_const` makes the tile index a generic +parameter (`tilezero tmm{t}`, `t = const T`). `amx_ops.rs` exposes the whole +`X86InstrAMX.td` surface that way, and its tests read the emitted bytes back +out of the text segment and pin them to this table — on any x86_64 host, no +EMR needed. Tile-operand aliasing (Gotcha 11) is a `const` assert, so `#UD` +is now a compile error. + +**The "mirror" (Gotcha 12, §4) is a reading of the byte table, not a hardware +quirk.** `MRMSrcReg4VOp3` puts `dst` in ModRM.reg, **S1 in ModRM.rm, S2 in +VEX.vvvv**, and Intel syntax names them in that order: `tdpbusd tmmD, tmmS1, +tmmS2` = `D += S1(M×K, plain) · S2(K×N, VNNI)`, with the `U`/`S` letters +naming S1 then S2. The table row `C4 E2 71 5E C2` (rm = tmm2, vvvv = tmm1) +is the mnemonic `tdpbusd tmm0, tmm2, tmm1` — `amx_ops::tdpbusd::<0, 2, 1>` — +which is exactly the kernel's placement (A u8 → tmm2, B VNNI i8 → tmm1). The +row's comment "dst=tmm0,vvvv=tmm1,rm=tmm2" had been read as the operand +list `(tmm0, tmm1, tmm2)`; the assembler reads `(tmm0, tmm1, tmm2)` as +rm = tmm1, vvvv = tmm2 = `C4 E2 69 5E C1`. Pinned two-sided in +`operand_order_is_intel_order_rm_then_vvvv`. + +Assembler-verified encodings, LLVM `Host.cpp` CPUID bits. The INT8/BF16 rows +are listed for reference and DID execute on EMR in the `(0,2,1)` placement; +the `(0,1,2)` placements shown here, and every row below them, have NOT +executed in this workspace (no GNR/DMR host): + +``` + CPUID bytes (tmm0,tmm1,tmm2 / tmm3,[rdi+rsi]) +TDPBSSD/TDPBSUD/TDPBUSD/TDPBUUD INT8 7.0:EDX[25] C4 E2 {6B,6A,69,68} 5E C1 +TDPBF16PS BF16 7.0:EDX[22] C4 E2 6A 5C C1 +TDPFP16PS FP16 7.1:EAX[21] C4 E2 6B 5C C1 +TCMMIMFP16PS / TCMMRLFP16PS COMPLEX 7.1:EDX[8] C4 E2 {69,68} 6C C1 +TDPBF8PS/TDPBHF8PS/TDPHBF8PS/ FP8 1E.1:EAX[4] C4 E5 {68,6B,6A,69} FD C1 (map5) + TDPHF8PS +TMMULTF32PS TF32 1E.1:EAX[6] C4 E2 69 48 C1 (dropped from LLVM main; 22.1.8 assembles the mnemonic, nightly LLVM 23 does not → emitted as raw bytes) +TILELOADDRS / TILELOADDRST1 MOVRS 1E.1:EAX[8] C4 E2 {7B,79} 4A 1C 37 +TCVTROWD2PS zmm0,tmm1,edi / ,3 AVX512 1E.1:EAX[7] 62 F2 46 48 4A C1 / 62 F3 7E 48 07 C1 03 (EVEX; needs avx512f cfg) +TCVTROWPS2{PHH,PHL,BF16H,BF16L} AVX512 62 F2 {44,46,47,45} 48 6D C1 +TILEMOVROW zmm0,tmm1,edi / ,5 AVX512 62 F2 45 48 4A C1 / 62 F3 7D 48 07 C1 05 +STTILECFG [rdi] / TILELOADDT1 TILE 7.0:EDX[24] C4 E2 79 49 07 / C4 E2 79 4B 14 16 +``` + +Detection: `amx_ops::amx_features()` (cached) returns the per-tier bits. +The execute gate for tile STATE is `simd_amx::amx_tile_available()` (TILE + +XCR0 + arch_prctl); `amx_available()` is that plus the INT8 bit and gates the +INT8 ops only; every other tier gates on `amx_tile_available()` AND its +`AmxFeatures` bit. `amx_report()` prints both gates and every tier bit. +Gotcha 14 (VM tile-state corruption) applies to every tier. + ## 6. Detection API (cached, CPU-aware) ```rust -use ndarray::simd::{amx_available, cpu_model, amx_report, CpuModel}; +use ndarray::simd::{amx_available, amx_tile_available, amx_features, cpu_model, amx_report, CpuModel}; -amx_available() // bool, cached once via LazyLock (the 4 gates of §1) +amx_available() // bool, cached once via LazyLock (the 4 gates of §1, INT8 bit last) +amx_tile_available() // the tier-agnostic tile gate (TILE + OSXSAVE + XCR0 + arch_prctl) +amx_features() // AmxFeatures — per-tier silicon bits (§5b) cpu_model() // CpuModel::{SapphireRapids,EmeraldRapids,GraniteRapids,SierraForest,OtherX86,NonX86} cpu_model().has_amx() // true for SPR/EMR/GNR; false for Sierra Forest (E-core) -amx_report() // e.g. "AMX [Emerald Rapids expects_amx=true]: TILE=true INT8=true BF16=true available=true" +amx_report() // e.g. "AMX [Emerald Rapids expects_amx=true]: TILE=true INT8=true BF16=true tile_available=true available=true | tiers: fp16=false complex=false fp8=false tf32=false avx512=false movrs=false" ``` Why `LazyLock`: the four gates (CPUID, XGETBV, one `arch_prctl`) are all diff --git a/.claude/knowledge/simd-codegen-oracle/README.md b/.claude/knowledge/simd-codegen-oracle/README.md index d5b58545..8bbe09d8 100644 --- a/.claude/knowledge/simd-codegen-oracle/README.md +++ b/.claude/knowledge/simd-codegen-oracle/README.md @@ -82,6 +82,28 @@ Measured on x86_64 + `x86-64-v3`, rustc 1.95.0. Full narrative in | `gather_lookup_u8` | 0 | 0 | `movzbl` chain, no arithmetic | | `serial_dependent_chain` | 0 | 27 | loop-carried dependency | +Group F — the mask family (PR #306), measured 2026-09-14 on rustc 1.98.1 +through the shipped library methods. Two runs: array polyfill first, then +after the four non-packed shapes were given backend-local AVX2 intrinsic +realizations. The bit-logic half needed nothing. + +| probe | polyfill run | after override | lowering | +|---|---|---|---| +| `ternlog_u64x8_maj3` / `_select` | 18 / 0 | unchanged | generated Shannon ladder → `vpand`/`vpandn`/`vpor`/`vpxor` | +| `ternlog_u32x16_xor_and` | 8 / 0 | unchanged | same, 32-bit lanes | +| `andnot_u64x8` | 6 / 0 | unchanged | `vpandn` | +| `popcnt_u64x8` | 21 / 0 | unchanged | `vpshufb` nibble-LUT popcount, not 8× `popcntq` | +| `xor_popcount_u64x8` | 25 / 0 | unchanged | `vpxor` + nibble popcount + add tree | +| **`rotate_left_lib_u64x8`** | **0 / 8** `rolq` | 10 / 2 (count setup) | `vpsllq`+`vpsrlq`+`vpor` per half — the earned u64 override | +| **`gt_bitmask_i32x16`** | **23 / 3** mixed (lanes 0, 13–15 peeled) | 9 / 0 | `vpcmpgtd` + movemask | +| **`cmpge_zero_mask_i32x16`** | **17 / 11** mixed | 10 / 0 | complemented sign-bit movemask | +| **`reduce_max_i32x16`** | **0 / 17** `cmpl` | 8 / 0 | `vpmaxsd` tree | + +The two "mixed" rows are the instructive ones: a shape can be *mostly* +packed and still carry a scalar peel, and the method's doc comment had +claimed a clean lowering it never had. Measure the shipped symbol, not the +look-alike. + **The headline:** LLVM vectorizes far more than intuition suggests — including cross-lane permutes, widening converts, and saturating arithmetic, all from plain scalar loops. It does **not** vectorize u64 diff --git a/.claude/knowledge/simd-codegen-oracle/baseline-x86_64-v3.toml b/.claude/knowledge/simd-codegen-oracle/baseline-x86_64-v3.toml index bc5929b0..5ac3009d 100644 --- a/.claude/knowledge/simd-codegen-oracle/baseline-x86_64-v3.toml +++ b/.claude/knowledge/simd-codegen-oracle/baseline-x86_64-v3.toml @@ -184,9 +184,91 @@ note = "MEASURED FULLY PACKED. Observed 81 packed / 0 scalar-lane-arith / 0 loop [probe.shiftor_rot_u64x8] expect = "unknown" -note = "MEASURED SCALAR -- 0 packed / 8 scalar-lane-arith / 19 memory, i.e. indistinguishable from rot_u64x8. Writing the rotate as an explicit (x >> n) | (x << (64-n)) does NOT persuade LLVM to vectorize it, so the refusal is about the 64-bit OPERATION and not about the rotate idiom. Driver asserts equality with u64::rotate_right, so this is the same function in a different spelling." +note = "MEASURED SCALAR on 2026-07 (0 packed / 8 scalar-lane-arith / 19 memory, indistinguishable from rot_u64x8): writing the rotate as an explicit (x >> n) | (x << (64-n)) did NOT persuade LLVM to vectorize it, so the refusal is about the 64-bit OPERATION and not about the rotate idiom. Driver asserts equality with u64::rotate_right, so this is the same function in a different spelling. OBSERVATION 2026-09-14, recorded rather than explained: in the run where U64x8::rotate_left/right became a vpsllq/vpsrlq/vpor intrinsic realization in simd_avx2.rs, THIS probe -- untouched -- read 10 packed / 2 scalar-lane-arith, byte-for-byte the library method's shape, while rot_u64x8 and shiftor_rot_const_u64x8 stayed 0 packed. The probe calls no library rotate; the mechanism (LLVM recognising the shift-or as the rotate operation now that a vector-typed lowering of that operation exists in the module and folding, or something else) is NOT established. Kept `unknown`; a probe whose reading moves without its source changing is a finding about the instrument, not the lane." [probe.shiftor_rot_const_u64x8] expect = "unknown" note = "MEASURED SCALAR -- 0 packed / 8 scalar-lane-arith / 8 memory, with BLAKE2b's COMPILE-TIME-CONSTANT 32/24/16/63. This is the sharper half: at u32 width a constant byte-granular amount folds to vpshufb and a constant bit-granular one folds to shift-or, both packed. At u64 width neither happens, for either kind of constant. A third independent confirmation that the u64 lane is the crate's one earned intrinsic override." + +# ============================================================================ +# Group F -- the MASK FAMILY (PR #306): the exact shapes `simd_masking_ops` +# calls on the x86-64-v3 (AVX2) backend, each through the LIBRARY method so +# the measurement is of shipped code. Rung 3 of the acceptance ladder, x86 +# arm. Measured 2026-09-14, rustc 1.98.1, in TWO runs: first on the pure +# array polyfill, then after the four shapes that did not lower packed were +# given backend-local intrinsic realizations in simd_avx2.rs. +# +# FIRST-RUN RESULT (array polyfill, no intrinsics): ternlog (both u64 tables +# and the u32 XOR_AND), andnot, popcnt and xor_popcount were ALREADY fully +# packed with 0 scalar lane arithmetic -- the oracle's standing finding +# ("scalar source, packed codegen") holds for the whole bit-logic half of the +# mask family, so those earned NO rewrite. Four shapes did not: rotate_left +# (0 packed / 8 rolq -- the known u64 rotate refusal, now confirmed on the +# shipped method), reduce_max (0 packed / 17 cmpl -- `iter().max()` is a +# scalar chain), and the two compare-to-bitmask forms, which were MIXED: +# LLVM vectorized lanes 1..=12 but peeled lanes 0 and 13..=15 into scalar +# compares plus shll/orl bit assembly (cmpge_zero_mask 17 packed / 11 scalar; +# gt_bitmask 23 / 3). Those four are the earned overrides; `expect` below is +# the SECOND-run state and is what the gate now holds. +# ============================================================================ + +[probe.ternlog_u64x8_maj3] +expect = "vectorized" +min_packed = 10 +max_scalar_lane_arith = 0 +note = "ARRAY POLYFILL, unchanged. Observed 18 packed / 0 scalar-lane-arith / 0 loop-control on both runs: the generated Shannon ladder (t0 != t1, general arm) lowers to vpand/vpandn/vpor/vpxor over both ymm halves. No intrinsic override earned." + +[probe.ternlog_u64x8_select] +expect = "vectorized" +min_packed = 10 +max_scalar_lane_arith = 0 +note = "ARRAY POLYFILL, unchanged. 0xCA (c ? a : b) -- the (T0 & !c) | (T1 & c) arm. Observed 18 packed / 0 scalar-lane-arith, identical shape to MAJ3." + +[probe.ternlog_u32x16_xor_and] +expect = "vectorized" +min_packed = 4 +max_scalar_lane_arith = 0 +note = "ARRAY POLYFILL, unchanged. The exact immediate simd_masking_ops uses. Observed 8 packed / 0 scalar-lane-arith." + +[probe.andnot_u64x8] +expect = "vectorized" +min_packed = 3 +max_scalar_lane_arith = 0 +note = "ARRAY POLYFILL, unchanged. `self & !other` folds to vpandn per half. Observed 6 packed / 0 scalar-lane-arith." + +[probe.popcnt_u64x8] +expect = "vectorized" +min_packed = 12 +max_scalar_lane_arith = 0 +note = "ARRAY POLYFILL, unchanged -- and the surprise of the group: eight `count_ones()` in a lane loop did NOT become eight scalar popcntq; LLVM emitted the vpshufb nibble-LUT popcount over both ymm halves (21 packed / 0 scalar-lane-arith). No override earned." + +[probe.xor_popcount_u64x8] +expect = "vectorized" +min_packed = 15 +max_scalar_lane_arith = 0 +note = "ARRAY POLYFILL, unchanged. vpxor + the same vpshufb popcount + a horizontal add tree. Observed 25 packed / 0 scalar-lane-arith." + +[probe.rotate_left_lib_u64x8] +expect = "vectorized" +min_packed = 6 +max_scalar_lane_arith = 2 +note = "INTRINSIC REALIZATION EARNED. First run (array polyfill): 0 packed / 8 rolq %cl -- the shipped method scalarised exactly like the hand-written rot_u64x8 mirror. Second run (vpsllq + vpsrlq + vpor per 256-bit half, uniform xmm count): 10 packed / 2 scalar-lane-arith / 1 loop-control. The 2 scalar ops are `andl $63` and `subl` on the COUNT (n % 64, 64 - n), not on lane data; the branch is the n == 0 early return. The u32 lane got this lowering for free; the u64 lane had to be told." + +[probe.gt_bitmask_i32x16] +expect = "vectorized" +min_packed = 5 +max_scalar_lane_arith = 0 +note = "INTRINSIC REALIZATION EARNED. First run (index loop): 23 packed / 3 scalar-lane-arith / 5 loop-control -- MIXED: LLVM vectorized lanes 1..=12 (a ymm at offset 4 and an xmm at offset 36) and peeled lanes 0 and 13..=15 into scalar cmpl + shll/orl bit assembly. The doc comment on the method had claimed a clean packed lowering; the measurement did not bear it out. Second run (vpcmpgtd per half + vmovmskps): 9 packed / 0 scalar-lane-arith / 0 loop-control -- LLVM further folded the two movemasks into one vpackssdw/vpacksswb/vpmovmskb." + +[probe.cmpge_zero_mask_i32x16] +expect = "vectorized" +min_packed = 6 +max_scalar_lane_arith = 0 +note = "INTRINSIC REALIZATION EARNED. First run (index loop): 17 packed / 11 scalar-lane-arith / 4 loop-control -- the same lane-0 + lanes-13..=15 peel as gt_bitmask. Second run (complemented vmovmskps sign bits per half): 10 packed / 0 scalar-lane-arith / 0 loop-control (vpcmpgtd against all-ones + pack + vpmovmskb)." + +[probe.reduce_max_i32x16] +expect = "vectorized" +min_packed = 5 +max_scalar_lane_arith = 0 +note = "INTRINSIC REALIZATION EARNED. First run (`iter().max()`): 0 packed / 17 cmpl -- a scalar compare chain, nothing packed. Second run (vpmaxsd tree 16 -> 8 -> 4 -> 2 -> 1): 8 packed / 0 scalar-lane-arith. reduce_min is the vpminsd twin, not separately probed (same shape, same ladder)." diff --git a/.claude/knowledge/simd-codegen-oracle/probes.rs b/.claude/knowledge/simd-codegen-oracle/probes.rs index 2f17dcdf..4cc5aa9e 100644 --- a/.claude/knowledge/simd-codegen-oracle/probes.rs +++ b/.claude/knowledge/simd-codegen-oracle/probes.rs @@ -42,7 +42,7 @@ //! - The return value is consumed (folded into the printed report), so nothing //! is dead-code-eliminated. -use ndarray::simd::{add_mul_f32, I8x32, U32x16, U64x4, U64x8, U8x64}; +use ndarray::simd::{add_mul_f32, ternlog, I32x16, I8x32, U32x16, U64x4, U64x8, U8x64}; use std::hint::black_box; use std::time::{SystemTime, UNIX_EPOCH}; @@ -602,6 +602,80 @@ pub fn arx_lane512_x8(a: [U32x16; 8], b: [U32x16; 8]) -> [U32x16; 8] { // Driver — runtime-derived inputs, every result consumed. // ============================================================================ +// ============================================================================ +// Group F — the MASK FAMILY (PR #306). Does the x86-64-v3 backend's array +// polyfill lower the exact shapes `simd_masking_ops` calls to packed AVX2, or +// does any of them scalarise the way the u64 rotate does? Every probe calls +// the LIBRARY method (no hand-written mirror) so the answer is about the code +// that ships, not about a look-alike. Rung 3 of the acceptance ladder, x86. +// ============================================================================ + +/// `U64x8::ternlog::` — the two-of-three majority, the table with the +/// deepest Shannon ladder (t0 != t1, neither zero nor full, not complements). +#[inline(never)] +pub fn ternlog_u64x8_maj3(a: U64x8, b: U64x8, c: U64x8) -> U64x8 { + a.ternlog::<{ ternlog::MAJ3 }>(b, c) +} + +/// `U64x8::ternlog::<0xCA>` — `c ? a : b` (bit select), the general-case +/// ladder arm `(T0 & !c) | (T1 & c)`. +#[inline(never)] +pub fn ternlog_u64x8_select(a: U64x8, b: U64x8, c: U64x8) -> U64x8 { + a.ternlog::<0xCA>(b, c) +} + +/// `U32x16::ternlog::` — the exact immediate `simd_masking_ops` +/// uses four times. +#[inline(never)] +pub fn ternlog_u32x16_xor_and(a: U32x16, b: U32x16, c: U32x16) -> U32x16 { + a.ternlog::<{ ternlog::XOR_AND }>(b, c) +} + +/// `U64x8::andnot` — `self & !other`, the mask set-difference. +#[inline(never)] +pub fn andnot_u64x8(a: U64x8, b: U64x8) -> U64x8 { + a.andnot(b) +} + +/// `U64x8::popcnt` — per-lane population count (no packed popcnt below +/// AVX-512 VPOPCNTDQ; the question is whether LLVM emits the pshufb-nibble +/// or Harley-Seal idiom, or eight scalar `popcntq`). +#[inline(never)] +pub fn popcnt_u64x8(a: U64x8) -> U64x8 { + a.popcnt() +} + +/// `U64x8::xor_popcount` — Hamming distance, the reduction form. +#[inline(never)] +pub fn xor_popcount_u64x8(a: U64x8, b: U64x8) -> u64 { + a.xor_popcount(b) +} + +/// `U64x8::rotate_left` through the LIBRARY method (rot_u64x8 above measures a +/// hand-written mirror of it; this one measures the shipped code). +#[inline(never)] +pub fn rotate_left_lib_u64x8(a: U64x8, n: u32) -> U64x8 { + a.rotate_left(n) +} + +/// `I32x16::gt_bitmask` — packed signed compare to a 16-bit LSB-first mask. +#[inline(never)] +pub fn gt_bitmask_i32x16(a: I32x16, b: I32x16) -> u16 { + a.gt_bitmask(b) +} + +/// `I32x16::cmpge_zero_mask` — sign-bit extraction as a 16-bit mask. +#[inline(never)] +pub fn cmpge_zero_mask_i32x16(a: I32x16) -> u16 { + a.cmpge_zero_mask() +} + +/// `I32x16::reduce_max` — horizontal max over the polyfill. +#[inline(never)] +pub fn reduce_max_i32x16(a: I32x16) -> i32 { + a.reduce_max() +} + /// Wall-clock nanoseconds XORed with argc — a runtime value no build-time /// constant folder can predict, used to seed a small PRNG for building probe /// inputs. Piped through `black_box` at every call site below as well, so @@ -796,5 +870,84 @@ fn main() { acc ^= n1.iter().fold(0u64, |s, v| s ^ v.reduce_sum() as u64); acc ^= n2.iter().fold(0u64, |s, v| s ^ v.reduce_sum() as u64); + // ---- Group F (mask family) ---- + let (fa, fb, fc) = ( + U64x8::from_array(std::array::from_fn(|_| rng.next())), + U64x8::from_array(std::array::from_fn(|_| rng.next())), + U64x8::from_array(std::array::from_fn(|_| rng.next())), + ); + // Correctness first: every ternlog probe is checked against the bit-serial + // truth-table definition before its codegen is reported. + let ref_ternlog = |imm: i32, a: u64, b: u64, c: u64| -> u64 { + let mut out = 0u64; + for bit in 0..64 { + let idx = (((a >> bit) & 1) << 2) | (((b >> bit) & 1) << 1) | ((c >> bit) & 1); + out |= (((imm as u64) >> idx) & 1) << bit; + } + out + }; + let maj = ternlog_u64x8_maj3(black_box(fa), black_box(fb), black_box(fc)); + let sel = ternlog_u64x8_select(black_box(fa), black_box(fb), black_box(fc)); + { + let (a, b, c) = (fa.to_array(), fb.to_array(), fc.to_array()); + for i in 0..8 { + assert_eq!(maj.to_array()[i], ref_ternlog(ternlog::MAJ3, a[i], b[i], c[i]), "MAJ3 lane {i}"); + assert_eq!(sel.to_array()[i], ref_ternlog(0xCA, a[i], b[i], c[i]), "0xCA lane {i}"); + } + } + acc ^= maj.reduce_sum() ^ sel.reduce_sum(); + + let (ua, ub, uc) = ( + U32x16::from_array(std::array::from_fn(|_| rng.next() as u32)), + U32x16::from_array(std::array::from_fn(|_| rng.next() as u32)), + U32x16::from_array(std::array::from_fn(|_| rng.next() as u32)), + ); + let xa = ternlog_u32x16_xor_and(black_box(ua), black_box(ub), black_box(uc)); + { + let (a, b, c) = (ua.to_array(), ub.to_array(), uc.to_array()); + for i in 0..16 { + assert_eq!( + xa.to_array()[i], + ref_ternlog(ternlog::XOR_AND, a[i] as u64, b[i] as u64, c[i] as u64) as u32, + "XOR_AND lane {i}" + ); + } + } + acc ^= xa.reduce_sum() as u64; + + let an = andnot_u64x8(black_box(fa), black_box(fb)); + assert_eq!(an.to_array(), std::array::from_fn::(|i| fa.to_array()[i] & !fb.to_array()[i])); + acc ^= an.reduce_sum(); + + let pc = popcnt_u64x8(black_box(fa)); + assert_eq!(pc.to_array(), std::array::from_fn::(|i| fa.to_array()[i].count_ones() as u64)); + acc ^= pc.reduce_sum(); + + let xp = xor_popcount_u64x8(black_box(fa), black_box(fb)); + assert_eq!(xp, (0..8).map(|i| (fa.to_array()[i] ^ fb.to_array()[i]).count_ones() as u64).sum::()); + acc ^= xp; + + let rn = 1 + (rng.next() % 63) as u32; + let rl = rotate_left_lib_u64x8(black_box(fa), black_box(rn)); + assert_eq!(rl.to_array(), std::array::from_fn::(|i| fa.to_array()[i].rotate_left(rn))); + acc ^= rl.reduce_sum(); + + let (ia, ib) = ( + I32x16::from_array(std::array::from_fn(|_| rng.next() as i32)), + I32x16::from_array(std::array::from_fn(|_| rng.next() as i32)), + ); + let gt = gt_bitmask_i32x16(black_box(ia), black_box(ib)); + let ge0 = cmpge_zero_mask_i32x16(black_box(ia)); + let mx = reduce_max_i32x16(black_box(ia)); + { + let (a, b) = (ia.to_array(), ib.to_array()); + let want_gt = (0..16).fold(0u16, |m, i| m | (((a[i] > b[i]) as u16) << i)); + let want_ge0 = (0..16).fold(0u16, |m, i| m | (((a[i] >= 0) as u16) << i)); + assert_eq!(gt, want_gt, "gt_bitmask"); + assert_eq!(ge0, want_ge0, "cmpge_zero_mask"); + assert_eq!(mx, *a.iter().max().unwrap(), "reduce_max"); + } + acc ^= (gt as u64) ^ ((ge0 as u64) << 16) ^ ((mx as u32 as u64) << 32); + println!("simd-codegen-oracle: probes executed, combined checksum = {acc:#018x}"); } diff --git a/.claude/knowledge/vertical-simd-consumer-contract.md b/.claude/knowledge/vertical-simd-consumer-contract.md index 3ab3bf1f..307f682f 100644 --- a/.claude/knowledge/vertical-simd-consumer-contract.md +++ b/.claude/knowledge/vertical-simd-consumer-contract.md @@ -423,3 +423,93 @@ removal commit had to be reverted once the step-1 gate was found. "Contains raw intrinsics" and "raw intrinsics are reachable" are different claims; audit the second. Full record: board `EPIPHANIES.md` 2026-07-28 entry + the `.cargo/config.toml` comment block. + +--- + +## The masking layer: `simd_masking_ops.rs` and the two laws above the backends (2026-09-13) + +Operator-ruled during the mask-RISC arc; recorded here because every W1a +primitive that turns values into masks, composes masks, or reduces under a +mask is now expected to land in THIS shape rather than beside `add_i8`. + +```text +consumers semantic ops only — TERNLOG, AND, XOR, COUNT +simd_masking_ops.rs slice/chunk/tail ergonomics, *_assign forms, + mask composition, masked reductions — never an ISA +simd.rs architecture-agnostic types, compile-time selection +simd_{avx512,avx2,neon,wasm,scalar}.rs peer backends, each owns realization +``` + +- **Polyfill law.** Every public mask/SIMD primitive a consumer uses has a + compile-time implementation in ALL FIVE backends; scalar is a peer, not a + fallback; no runtime ISA dispatch above `simd.rs`; hardware-specific + optimisation (including truth-table specialisation of `ternlog`) lives in + the backend file only. A consumer that branches on ISA is a violation. +- **Backend law.** No shared generic implementation body under the backends. + Shared tests and shared *generated* truth-table logic are fine + (`tools/gen_ternlog_bodies.py` emits backend-LOCAL bodies between + `GEN-TERNLOG` markers); a common function the backends call into is not. +- **Placement rule for new work.** Backend semantics (what `U64x8::ternlog` + *is*) never move into `simd_masking_ops.rs`; slice ergonomics, tail + handling, reusable-destination forms and fused conveniences never move into + a backend. A facade function that cannot be one delegation is the signal + the substrate is missing a word (the missing-capability STOP rule). +- **Acceptance for a mask primitive** adds one row to the criteria above: + the cross-ISA parity harnesses (`crates/wasm-simd-parity`, run under node; + `crates/neon-simd-parity`, run under qemu) must carry the primitive's + check — the x86 `cargo test` suite never compiles `simd_wasm.rs` or + `simd_neon.rs`, so a backend body that only x86 tests cover is unproven on + the target it was written for. The 256-table `ternlog` arm is the template. +- **AArch64 without the hardware — the acceptance ladder.** A NEON body is + authored from the LLVM/Clang intrinsic corpus + Rust `core::arch::aarch64` + declarations and proven by: (1) cross-target compile; (2) the parity harness + under qemu (CI); (3) cross-compiled assembly showing the expected NEON ops + and no unexpected scalarisation — a `to_array()`-per-lane loop passed rungs + 1, 2 and 4 and FAILED 3 (536 scalar vs 4 vector ops), which is why rung 3 is + not optional; (4) exhaustive reference parity; (5) hardware benchmarking as + a later performance gate. Command shape for rung 3: + `cargo rustc --release --manifest-path crates/neon-simd-parity/Cargo.toml + --target aarch64-unknown-linux-gnu -- --emit=asm`, then count + `(and|orr|eor|bic|orn) v*.16b` against `(and|orr|eor|bic) w*,`. +- **`unsafe` at the intrinsic boundary — where and why (measured, 1.98.1, + `tools/safe_intrinsic_probe`).** x86 and aarch64 SIMD intrinsics are safe + fns whose CALL requires the caller to carry the matching + `#[target_feature]`; build-config features do not count (rustc says so in + the E0133 note), and a safe annotated fn called from a plain fn fails the + same way. The per-fn annotation is not the fix: a `simd_{arch}.rs` file is + compiled for exactly one target CPU, selected by `cfg`, so the feature is + already a property of the file — restating it on every fn is illogical and + propagates to every safe caller; rustc just does not read the `cfg` as + evidence. wasm32 + simd128 intrinsics are callable from plain safe code. Rule: a backend + method owns exactly one expression-narrow `unsafe` at its intrinsic + boundary with a SAFETY line; wasm bodies carry none; nothing above a + backend file is ever `unsafe`. Re-run the probe after a toolchain bump — + the day rustc counts baseline features, the aarch64/x86 rows flip and the + blocks come out. +- **Five execution flavours, one semantic surface (operator, 2026-09-13).** + (1) x86-64-v3 default/CI → `simd_avx2`; (2) AVX-512/v4 → `simd_avx512`; + (3) `target-cpu=native` → backend chosen from the build host's CPUID; + (4) `nightly-simd` → `core::simd`; (5) `runtime-dispatch` → LazyLock + detection then a specialised kernel. `#[target_feature]` propagation is not + the architecture: the selected backend (or the LazyLock branch) is the + capability proof, intrinsics stay at the backend's narrow `unsafe` + boundary, and `simd_masking_ops` / mask-RISC / consumers never inherit an + ISA calling contract. A mask primitive is never routed through Scalar + because rustc wants `unsafe` at an intrinsic. **Audit rule:** a new mask + primitive is proven on every flavour whose backend has a native lane type + for it — check `simd.rs`'s re-export arm per target, not the file you + authored in; `U64x8`/`I32x16` resolved to scalar on aarch64 and wasm32 + until the #306 audit caught it. +- **Measure the shipped symbol before overriding it (2026-09-14, the AVX2 + arm of the mask family).** The plan said "replace the `avx2_int_type!` + array polyfills with native `[__m256i; 2]`"; the codegen oracle + (`.claude/knowledge/simd-codegen-oracle/`, Group F) said six of the ten + mask shapes — every ternlog ladder, andnot, popcnt, xor_popcount — were + ALREADY packed from scalar source, and four were not (u64 rotate, i32 + horizontal min/max, and the two compare-to-bitmask forms, which were + *mixed*: mostly packed with lanes 0 and 13–15 peeled to scalar). Only the + four got intrinsic realizations. Rule: a polyfill lane loop is not scalar + because it is spelled as a loop; it is scalar when `--emit asm` on the + shipped method says so — and "mostly packed" is a category the oracle + must be able to report, because a peel is invisible to any parity test. diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b05b80ac..6a6436dc 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -250,18 +250,45 @@ jobs: # to host build scripts. Combined with explicit `--target` (so cargo # distinguishes host from target even when they're the same triple), # this gives us "v4 for our crate, baseline for build scripts." + # + # # Why `--config .cargo/config-v4.toml` and NOT that env var (2026-09-14): + # + # cargo JOINS every matching `target..rustflags` and + # `target..rustflags` entry, and the last `-Ctarget-cpu` wins. + # Measured with `cargo -v`: the env var form passed `x86-64-v4` and THEN + # `.cargo/config.toml`'s cfg-keyed `x86-64-v3`, so this job had been + # checking the AVX2 arm while named "tier4-avx512-check" — the codegen + # witness on PR #306 found 0 `vpternlog` in a "v4" build. `--config` + # is the same cfg key at higher precedence, placed LAST, so v4 wins; + # explicit `--target` still keeps build scripts on the host baseline. + # + # # ...and why `env -u RUSTFLAGS` (first run of the assertion below, e730109): + # + # This workflow sets a GLOBAL `RUSTFLAGS: "-D warnings"`, and a RUSTFLAGS + # env REPLACES every `.cargo/config*` rustflags entry outright (cargo's + # precedence: RUSTFLAGS > target. > target. > build). So with + # the global env in force NO config target-cpu applies — `--config` or + # not — and the probe assembled to 0 vpternlog. The steps below unset it + # for the cargo call; `-Dwarnings` comes back through config-v4.toml's + # own rustflags list so the warning gate is not lost. (The same global + # env also erases the v3 pin and the dalek/poly1305 cfgs for every other + # x86 job — a pre-existing gap, recorded in the blackboard, not this PR's.) runs-on: ubuntu-latest name: tier4-avx512-check - env: - CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUSTFLAGS: "-D warnings -Ctarget-cpu=x86-64-v4" steps: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 - name: cargo check (v4 / AVX-512 dispatch arm) - run: cargo check --target=x86_64-unknown-linux-gnu -p ndarray --features approx,serde,rayon + run: env -u RUSTFLAGS cargo --config .cargo/config-v4.toml check --target=x86_64-unknown-linux-gnu -p ndarray --features approx,serde,rayon - name: cargo check (v4 / AVX-512 + hpc-extras) - run: cargo check --target=x86_64-unknown-linux-gnu -p ndarray --features approx,serde,rayon,hpc-extras + run: env -u RUSTFLAGS cargo --config .cargo/config-v4.toml check --target=x86_64-unknown-linux-gnu -p ndarray --features approx,serde,rayon,hpc-extras + - name: prove the arm actually selected (a v4 build must emit vpternlog) + # Runs the witness script in asm-only mode (this runner may lack + # avx512f, so the probe is never executed here) rather than an inline + # copy: the script owns the stale-assembly guard (`rm -f` + `touch` of + # the probe source, without which a rust-cache hit re-emits no `.s`). + run: env -u RUSTFLAGS WITNESS_NO_RUN=1 CARGO_ARGS='--config .cargo/config-v4.toml' bash scripts/codegen-witness.sh avx512 nightly-simd-polyfill: # TD-SIMD-9 from .claude/knowledge/simd-dispatch-architecture.md. diff --git a/.github/workflows/simd-matrix.yaml b/.github/workflows/simd-matrix.yaml new file mode 100644 index 00000000..09535a0e --- /dev/null +++ b/.github/workflows/simd-matrix.yaml @@ -0,0 +1,213 @@ +on: + pull_request: + paths: + - 'src/simd*.rs' + - 'src/simd_nightly/**' + - 'src/simd_masking_ops.rs' + - 'src/hpc/amx_ops.rs' + - 'src/hpc/amx_matmul.rs' + - 'crates/simd-masking-parity/**' + - 'crates/neon-simd-parity/**' + - 'examples/ternlog_codegen_probe.rs' + - 'examples/amx_realization_report.rs' + - 'scripts/masking-parity.sh' + - 'scripts/codegen-witness.sh' + - 'scripts/neon-asm-rung3.sh' + - 'tools/gen_ternlog_bodies.py' + - '.cargo/**' + - 'Cargo.toml' + - 'src/lib.rs' + - '.github/workflows/simd-matrix.yaml' + merge_group: + push: + branches: + - master + - main + +name: SIMD realization matrix + +# Least privilege: nothing here pushes, comments, or releases. Every job +# only reads the tree, so the token is read-only and is not persisted into +# the checkout's git config (a repository-controlled command that runs after +# checkout would otherwise inherit whatever the repo's default token can do). +permissions: + contents: read + +# Two axes, one program. +# +# realization × platform +# avx512 / avx2 / neon / wasm / scalar / nightly × x86_64 / aarch64 / wasm32 +# +# Four rows run the SAME facade-only parity program unconditionally +# (`crates/simd-masking-parity`, via `scripts/masking-parity.sh `) — it +# has no idea which backend `simd.rs` selected, so a row proves "this +# realization is bit-identical to its scalar / bit-serial references" and +# nothing else. The avx512 row runs it ONLY on a runner that has avx512f; +# otherwise it degrades to an assembly-only assertion, and on that path the +# AVX-512 realization's bits are NOT proven in CI (the local v4 gate is the +# record for them). Where a row's assembly can be inspected, the tiny opt-3 +# codegen oracle (`examples/ternlog_codegen_probe.rs`, via +# `scripts/codegen-witness.sh `) runs beside it: the parity program +# proves bits at the parity crate's release opt-level 2, the oracle proves the +# backend selected the instruction it is REQUIRED to select at opt-level 3. +# Neither replaces the other. +# +# The scalar realization has no host of its own: it is what `simd.rs` selects +# on wasm32 WITHOUT `+simd128`, so the `scalar` row is a wasm32 build with the +# feature off, run under node. `nightly` is the `core::simd` realization +# behind the opt-in `nightly-simd` feature and needs a nightly rustc. +# +# No workflow-global RUSTFLAGS here, on purpose: a global RUSTFLAGS REPLACES +# every cargo-config `rustflags` entry, which is how the v4 row would silently +# become a v3 row (see `.github/workflows/ci.yaml` tier4 for the incident). +# The v4 row passes `--config .cargo/config-v4.toml` through CARGO_ARGS. + +env: + CARGO_TERM_COLOR: always + +jobs: + native: + # x86_64 host at the crate's default target-cpu (x86-64-v3 = the AVX2 + # realization). Also the ONLY row that can exercise AMX: the tile ops are + # runtime-gated and always compiled into native builds, so the report + # prints which gates this runner clears (`tile_available`/`available` + # false on a non-AMX runner is the expected, honest answer) and the + # encoding tests pin the assembled bytes without executing a tile op. + runs-on: ubuntu-latest + name: realization/avx2 × x86_64 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - name: generated ternlog bodies are current + run: python3 tools/gen_ternlog_bodies.py --check + - name: masking parity (native, default config) + run: bash scripts/masking-parity.sh native + - name: codegen witness (avx2) + run: bash scripts/codegen-witness.sh avx2 + - name: AMX realization report (runtime-gated; prints this runner's gates) + run: cargo run --example amx_realization_report + - name: AMX encoding + detection tests (no tile op executes) + run: cargo test --lib -- hpc::amx_ops simd_amx + + native-v4: + # Same host, AVX-512 realization via the v4 cargo config. Building and + # inspecting the assembly needs no AVX-512 silicon; RUNNING the parity + # binary and the probe's self-check does, and GitHub's ubuntu runners do + # not promise it — so the run steps are gated on /proc/cpuinfo and report + # SKIPPED loudly rather than SIGILL. The build + witness inspection (which + # asserts vpternlog was selected) always runs. + runs-on: ubuntu-latest + name: realization/avx512 × x86_64 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + key: v4 + - name: detect the full x86-64-v4 AVX-512 set on this runner + id: cpu + # A v4 build may emit any of F/BW/CD/DQ/VL (the masking ops use the + # BW/VL byte and word compares beside the F ternlog), so avx512f alone + # would let a partially-capable host reach the run steps and SIGILL. + # All five or none. + run: | + has=1 + for f in avx512f avx512bw avx512cd avx512dq avx512vl; do + grep -q -w "$f" /proc/cpuinfo || { echo "missing: $f"; has=0; } + done + echo "has=$has" >> "$GITHUB_OUTPUT" + grep -m1 'model name' /proc/cpuinfo || true + - name: build the parity program at x86-64-v4 + run: env -u RUSTFLAGS cargo --config .cargo/config-v4.toml build --release --manifest-path crates/simd-masking-parity/Cargo.toml --bin simd-masking-parity --target x86_64-unknown-linux-gnu + - name: masking parity (native, v4 config) + if: steps.cpu.outputs.has == '1' + run: CARGO_ARGS='--config .cargo/config-v4.toml' bash scripts/masking-parity.sh native + - name: codegen witness (avx512) — assembly inspection + native self-check + if: steps.cpu.outputs.has == '1' + run: CARGO_ARGS='--config .cargo/config-v4.toml' bash scripts/codegen-witness.sh avx512 + - name: codegen witness (avx512) — assembly inspection only (runner lacks avx512f) + if: steps.cpu.outputs.has == '0' + # The SAME script, in its asm-only mode — one implementation of the + # stale-assembly guard (`rm -f` + `touch`) and of the symbol + # attribution, not a second hand-rolled copy that drifts. + run: | + echo "::warning::runner lacks avx512f — v4 parity run and probe self-check SKIPPED; asserting the emitted assembly only" + env -u RUSTFLAGS WITNESS_NO_RUN=1 CARGO_ARGS='--config .cargo/config-v4.toml' bash scripts/codegen-witness.sh avx512 + + neon: + # aarch64 realization: cross-build on the x86 runner, run under qemu-user. + # Three rungs: parity under qemu (bits), the codegen witness (opt-3 NEON + # logic on v*.16b, GPR logic bounded), and rung 3 of the pre-existing NEON + # asm gate (`neon-simd-parity`, the wider type surface). + runs-on: ubuntu-latest + name: realization/neon × aarch64 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@stable + with: + targets: aarch64-unknown-linux-gnu + - run: rustup target add aarch64-unknown-linux-gnu + - uses: Swatinem/rust-cache@v2 + with: + key: aarch64 + - name: install aarch64 cross toolchain + qemu-user + run: sudo apt-get update && sudo apt-get install -y gcc-aarch64-linux-gnu qemu-user-static + - name: masking parity (neon, qemu) + run: bash scripts/masking-parity.sh neon-qemu + - name: codegen witness (neon) + run: CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc bash scripts/codegen-witness.sh neon aarch64-unknown-linux-gnu + - name: NEON asm rung 3 + run: CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc bash scripts/neon-asm-rung3.sh + + wasm: + # wasm32 with +simd128 = the `simd_wasm` realization; wasm32 WITHOUT it is + # what `simd.rs` selects as the scalar realization — the scalar backend's + # only executable row, run through the identical program. + runs-on: ubuntu-latest + name: realization/wasm + scalar × wasm32 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@stable + with: + targets: wasm32-unknown-unknown + - run: rustup target add wasm32-unknown-unknown + - uses: Swatinem/rust-cache@v2 + with: + key: wasm + - uses: actions/setup-node@v4 + with: + node-version: "22" + - name: masking parity (wasm, +simd128) + run: bash scripts/masking-parity.sh wasm + - name: masking parity (scalar realization = wasm32 without simd128) + run: bash scripts/masking-parity.sh wasm-scalar + + nightly: + # The `core::simd` realization behind the opt-in `nightly-simd` feature. + # Same program, same reference, nightly rustc; plus the lib tests that + # exercise the arm directly (masking ops, facade tests, AMX encodings — + # the latter because nightly's newer LLVM is where a dropped mnemonic + # first surfaces, as TF32 did). + runs-on: ubuntu-latest + name: realization/nightly × x86_64 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@nightly + - uses: Swatinem/rust-cache@v2 + with: + key: nightly + - name: masking parity (nightly-simd) + run: bash scripts/masking-parity.sh nightly + - name: lib tests on the nightly arm + run: cargo +nightly test --lib --features nightly-simd -- simd_masking_ops simd::tests hpc::amx_ops simd_amx diff --git a/CLAUDE.md b/CLAUDE.md index bb7490e4..f2aad0e9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -6,7 +6,8 @@ - **What**: High-performance linear algebra with pluggable BLAS backends (Native SIMD, MKL, OpenBLAS) - **Source**: `adaworldapi/rustynum` — reference GEMM, SIMD, and FFI implementations - **Target**: This repo — ndarray fork enhanced with HPC backends -- **Rust**: 1.94 Stable only. No nightly features. +- **Rust**: stable only — the pinned `rust-toolchain.toml` (1.98.1; `rust-version` in `Cargo.toml` is the floor). No nightly features on any default or supported build path. + - **The one documented exception — `nightly-simd` (opt-in, validation-only, since PR #173):** a Cargo feature that swaps the SIMD realization for `core::simd` (`src/simd_nightly/*`, `#![feature(portable_simd)]`) so the realization matrix can witness that backend too. It is never enabled by default, nothing on stable may depend on it, every stable CI row builds without it, and it is exercised only by the dedicated nightly CI rows (`nightly-simd-polyfill`, the `simd-matrix` nightly row) and `scripts/masking-parity.sh nightly`. Removing the feature would drop that backend from the matrix; enabling it anywhere by default would violate this rule. ## Agent Protocol This project uses specialized agents in `.claude/agents/`. Follow these rules: diff --git a/Cargo.toml b/Cargo.toml index 11ba2ddb..2b9c5fba 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -59,6 +59,14 @@ required-features = ["std"] name = "ternlog_amortization_probe" required-features = ["std"] +[[example]] +name = "ternlog_codegen_probe" +required-features = ["std"] + +[[example]] +name = "amx_realization_report" +required-features = ["std"] + [[example]] name = "amx_gemm_bench" required-features = ["std"] @@ -452,6 +460,7 @@ exclude = [ "crates/burn", "crates/wasm-simd-parity", "crates/neon-simd-parity", + "crates/simd-masking-parity", # Cross-repo: its dev-dep is a PATH into a lance-graph sibling checkout. # In-workspace, a missing sibling would fail resolution for EVERY member. "crates/sigker-parity", @@ -484,6 +493,19 @@ cblas-sys = { version = "0.1.4", default-features = false } [profile.bench] debug = true +# The codegen-witness profile (`scripts/codegen-witness.sh`): optimized so LLVM +# actually performs the vectorization the SIMD realization matrix certifies, +# with no debuginfo and no LTO so it stays a TINY build of one example, not a +# release build of the crate. The semantic arms of the matrix run at opt-level +# 0 instead — bits are proven cheaply there, machine code is proven here. +[profile.ci-codegen] +inherits = "release" +opt-level = 3 +debug = 0 +lto = false +incremental = false +panic = "abort" + [profile.test.package.numeric-tests] opt-level = 2 [profile.test.package.blas-tests] diff --git a/crates/neon-simd-parity/src/main.rs b/crates/neon-simd-parity/src/main.rs index 9477ac27..66aba37a 100644 --- a/crates/neon-simd-parity/src/main.rs +++ b/crates/neon-simd-parity/src/main.rs @@ -18,7 +18,7 @@ fn main() { eprintln!("neon-simd-parity FAILED: lane/op code = {rc}"); std::process::exit(rc as i32); } - println!("neon-simd-parity OK: U32x16 / F32x16 / I8x16 lanes bit-identical to scalar"); + println!("neon-simd-parity OK: U32x16 / F32x16 / I8x16 lanes, ternlog (256 tables, U32x16 + U64x8), U64x8 algebra, I32x16 compares — bit-identical to scalar"); } #[cfg(not(target_arch = "aarch64"))] { @@ -41,6 +41,15 @@ mod checks { if let Err(code) = check_i8x16() { return code; } + if let Err(code) = check_ternlog_all_tables() { + return code; + } + if let Err(code) = check_u64x8_algebra() { + return code; + } + if let Err(code) = check_i32x16_compare() { + return code; + } 0 } @@ -84,9 +93,7 @@ mod checks { /// `F32x16` — the float hot-path lane (splat / roundtrip / add / reduce_sum). fn check_f32x16() -> Result<(), u32> { - let data: [f32; 16] = [ - 0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, - ]; + let data: [f32; 16] = [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0]; if F32x16::from_array(data).to_array() != data { return Err(20); } @@ -112,7 +119,9 @@ mod checks { if I8x16::from_array(a_arr).to_array() != a_arr { return Err(30); } - let sum = I8x16::from_array(a_arr).add(I8x16::from_array(b_arr)).to_array(); + let sum = I8x16::from_array(a_arr) + .add(I8x16::from_array(b_arr)) + .to_array(); for i in 0..16 { if sum[i] != a_arr[i].wrapping_add(b_arr[i]) { return Err(31); @@ -120,4 +129,318 @@ mod checks { } Ok(()) } + + /// `ternlog::` over ALL 256 truth tables, on BOTH lane types this target + /// resolves through `ndarray::simd`: the native `U32x16` (the generated + /// backend-local body in this tier's `simd_*.rs`) and `U64x8` (the scalar + /// backend re-exported on non-x86 targets — so the scalar body is proven on a + /// real non-x86 build rather than assumed). The reference is bit-serial and + /// lives here, independent of every backend. This is the cross-ISA half of the + /// acceptance matrix "reference == AVX-512 == AVX2 == NEON == WASM == scalar": + /// the x86 arms run under `cargo test` (`simd::tests::w1a9_*`), this tier's + /// arm runs here. + /// + /// Operands: the canonical `F0/CC/AA` triple (every one of the 8 index + /// combinations appears in every byte) plus a second triple with mixed lane + /// values so a per-lane transposition would be caught as well. Return codes: + /// `40 + ` nothing lane-specific — the failing table is reported through the + /// code `0x100 | IMM` for `U32x16` and `0x200 | IMM` for `U64x8`, so a CI + /// failure names the exact truth table. + fn check_ternlog_all_tables() -> Result<(), u32> { + use ndarray::simd::U64x8; + + fn reference_u64(imm: u32, a: u64, b: u64, c: u64) -> u64 { + let mut r = 0u64; + for bit in 0..64 { + let idx = (((a >> bit) & 1) << 2) | (((b >> bit) & 1) << 1) | ((c >> bit) & 1); + r |= (((imm >> idx) & 1) as u64) << bit; + } + r + } + + // Two operand triples per lane width. Triple 0 is the canonical + // all-8-minterms pattern; triple 1 mixes per-lane values. + let u32_triples: [([u32; 16], [u32; 16], [u32; 16]); 2] = [ + ([0xF0F0_F0F0; 16], [0xCCCC_CCCC; 16], [0xAAAA_AAAA; 16]), + ( + [ + 0x0000_0000, 0xFFFF_FFFF, 0x0000_0001, 0x8000_0000, 0x1234_5678, 0x9ABC_DEF0, 0xDEAD_BEEF, + 0xCAFE_BABE, 0x0F0F_0F0F, 0xF0F0_F0F0, 0x5555_5555, 0xAAAA_AAAA, 0x0000_00FF, 0xFF00_0000, + 0x0101_0101, 0x8080_8080, + ], + [ + 0x9E37_79B9, 0x1111_1111, 0xDEAD_C0DE, 0x0BAD_F00D, 0x7FFF_FFFF, 0x0000_0000, 0xFFFF_FFFF, + 0x1357_9BDF, 0x2468_ACE0, 0xFEDC_BA98, 0x0000_0010, 0x0000_001F, 0xABCD_EF01, 0x1020_4080, + 0x0F0F_F0F0, 0xC0DE_CAFE, + ], + [ + 0x1357_9BDF, 0xC0DE_CAFE, 0x0000_0000, 0xFFFF_FFFF, 0xA5A5_A5A5, 0x5A5A_5A5A, 0x0000_8000, + 0x8000_0001, 0x7777_7777, 0x8888_8888, 0x0F0F_0F0F, 0xF0F0_F0F0, 0x1234_5678, 0x8765_4321, + 0xFFFF_0000, 0x0000_FFFF, + ], + ), + ]; + let u64_triples: [([u64; 8], [u64; 8], [u64; 8]); 2] = [ + ([0xF0F0_F0F0_F0F0_F0F0; 8], [0xCCCC_CCCC_CCCC_CCCC; 8], [0xAAAA_AAAA_AAAA_AAAA; 8]), + ( + [ + 0, + u64::MAX, + 1, + 1 << 63, + 0x1234_5678_9ABC_DEF0, + 0xDEAD_BEEF_CAFE_BABE, + 0x0F0F_0F0F_F0F0_F0F0, + 0x5555_AAAA_5555_AAAA, + ], + [ + 0x9E37_79B9_7F4A_7C15, + 0x1111_1111_1111_1111, + 0xDEAD_C0DE_0BAD_F00D, + u64::MAX, + 0, + 0x1357_9BDF_2468_ACE0, + 0xFEDC_BA98_7654_3210, + 0xC0DE_CAFE_C0DE_CAFE, + ], + [ + 0x1357_9BDF_1357_9BDF, + 0, + u64::MAX, + 0xA5A5_A5A5_5A5A_5A5A, + 0x8000_0000_0000_0001, + 0x7777_7777_8888_8888, + 0xFFFF_0000_0000_FFFF, + 0x0123_4567_89AB_CDEF, + ], + ), + ]; + + macro_rules! check_imm { + ($imm:expr) => {{ + const IMM: i32 = $imm; + for (a, b, c) in &u32_triples { + let got = U32x16::from_array(*a) + .ternlog::(U32x16::from_array(*b), U32x16::from_array(*c)) + .to_array(); + for i in 0..16 { + let want = reference_u64(IMM as u32, a[i] as u64, b[i] as u64, c[i] as u64) as u32; + if got[i] != want { + return Err(0x100 | IMM as u32); + } + } + } + for (a, b, c) in &u64_triples { + let got = U64x8::from_array(*a) + .ternlog::(U64x8::from_array(*b), U64x8::from_array(*c)) + .to_array(); + for i in 0..8 { + if got[i] != reference_u64(IMM as u32, a[i], b[i], c[i]) { + return Err(0x200 | IMM as u32); + } + } + } + }}; + } + // 16 × 16 = all 256 tables, each a distinct monomorphization. + macro_rules! check_row { + ($hi:expr) => { + check_imm!($hi * 16 + 0); + check_imm!($hi * 16 + 1); + check_imm!($hi * 16 + 2); + check_imm!($hi * 16 + 3); + check_imm!($hi * 16 + 4); + check_imm!($hi * 16 + 5); + check_imm!($hi * 16 + 6); + check_imm!($hi * 16 + 7); + check_imm!($hi * 16 + 8); + check_imm!($hi * 16 + 9); + check_imm!($hi * 16 + 10); + check_imm!($hi * 16 + 11); + check_imm!($hi * 16 + 12); + check_imm!($hi * 16 + 13); + check_imm!($hi * 16 + 14); + check_imm!($hi * 16 + 15); + }; + } + check_row!(0); + check_row!(1); + check_row!(2); + check_row!(3); + check_row!(4); + check_row!(5); + check_row!(6); + check_row!(7); + check_row!(8); + check_row!(9); + check_row!(10); + check_row!(11); + check_row!(12); + check_row!(13); + check_row!(14); + check_row!(15); + Ok(()) + } + + /// `U64x8` algebra on this tier's NATIVE lane type against inline scalar `u64` + /// arithmetic — the bulk mask ops (`mask_and/or/xor/andnot`) ride exactly + /// these operators. Operands populate all-zero, all-one, the sign bit and + /// mixed patterns so a lane transposition or a wrong and-not polarity fails. + /// Return codes `0x300 + op`. + fn check_u64x8_algebra() -> Result<(), u32> { + use ndarray::simd::U64x8; + let a_arr: [u64; 8] = [ + 0, + u64::MAX, + 1 << 63, + 0xF0F0_F0F0_F0F0_F0F0, + 0x1234_5678_9ABC_DEF0, + 1, + 0xDEAD_BEEF_CAFE_BABE, + 0x5555_AAAA_5555_AAAA, + ]; + let b_arr: [u64; 8] = [ + u64::MAX, + 0, + 1 << 63, + 0xCCCC_CCCC_CCCC_CCCC, + 0x0FED_CBA9_8765_4321, + 1, + 0xC0DE_CAFE_C0DE_CAFE, + 0xAAAA_5555_AAAA_5555, + ]; + let (a, b) = (U64x8::from_array(a_arr), U64x8::from_array(b_arr)); + if a.to_array() != a_arr { + return Err(0x300); + } + let mut back = [0u64; 8]; + U64x8::from_slice(&a_arr).copy_to_slice(&mut back); + if back != a_arr { + return Err(0x301); + } + let (and, or, xor, not, andnot) = + ((a & b).to_array(), (a | b).to_array(), (a ^ b).to_array(), (!a).to_array(), a.andnot(b).to_array()); + let pop = a.popcnt().to_array(); + let mut hamming = 0u64; + for i in 0..8 { + if and[i] != a_arr[i] & b_arr[i] { + return Err(0x302); + } + if or[i] != a_arr[i] | b_arr[i] { + return Err(0x303); + } + if xor[i] != a_arr[i] ^ b_arr[i] { + return Err(0x304); + } + if not[i] != !a_arr[i] { + return Err(0x305); + } + if andnot[i] != a_arr[i] & !b_arr[i] { + return Err(0x306); + } + if pop[i] != a_arr[i].count_ones() as u64 { + return Err(0x307); + } + hamming += (a_arr[i] ^ b_arr[i]).count_ones() as u64; + } + if a.xor_popcount(b) != hamming { + return Err(0x308); + } + for &n in &[0u32, 1, 7, 31, 32, 63] { + let (l, r) = (a.rotate_left(n).to_array(), a.rotate_right(n).to_array()); + for i in 0..8 { + if l[i] != a_arr[i].rotate_left(n) { + return Err(0x309); + } + if r[i] != a_arr[i].rotate_right(n) { + return Err(0x30A); + } + } + } + let mut sum = 0u64; + for &x in &a_arr { + sum = sum.wrapping_add(x); + } + if a.reduce_sum() != sum { + return Err(0x30B); + } + if (a + b).to_array() != core::array::from_fn(|i| a_arr[i].wrapping_add(b_arr[i])) { + return Err(0x30C); + } + if (a - b).to_array() != core::array::from_fn(|i| a_arr[i].wrapping_sub(b_arr[i])) { + return Err(0x30D); + } + if !(a == U64x8::from_array(a_arr)) || a == b { + return Err(0x30E); + } + Ok(()) + } + + /// `I32x16` signed compares and arithmetic on this tier's NATIVE lane type + /// against inline scalar `i32` — `gt_bitmask` is the whole ordered-compare + /// family's primitive (`gt/lt/ge/le_i32_to_mask`). Operands include + /// `i32::MIN`, `i32::MAX`, `0`, `-1` and lanes that compare EQUAL, and the + /// `u16` bit order (bit i = lane i) is asserted lane by lane. Return codes + /// `0x400 + op`. + fn check_i32x16_compare() -> Result<(), u32> { + use ndarray::simd::I32x16; + let a_arr: [i32; 16] = + [i32::MIN, i32::MAX, 0, -1, 1, -2, 7, 7, 100, -100, 0x7FFF_0000, -0x7FFF_0000, 42, -42, 2, -3]; + let b_arr: [i32; 16] = + [i32::MIN, i32::MAX, 0, 0, -1, -2, 7, 8, -100, 100, -0x7FFF_0000, 0x7FFF_0000, 41, -41, -3, 2]; + let (a, b) = (I32x16::from_array(a_arr), I32x16::from_array(b_arr)); + if a.to_array() != a_arr { + return Err(0x400); + } + let gt = a.gt_bitmask(b); + let ge0 = a.cmpge_zero_mask(); + for i in 0..16 { + if ((gt >> i) & 1 == 1) != (a_arr[i] > b_arr[i]) { + return Err(0x401); + } + if ((ge0 >> i) & 1 == 1) != (a_arr[i] >= 0) { + return Err(0x402); + } + } + if a.simd_min(b).to_array() != core::array::from_fn(|i| a_arr[i].min(b_arr[i])) { + return Err(0x403); + } + if a.simd_max(b).to_array() != core::array::from_fn(|i| a_arr[i].max(b_arr[i])) { + return Err(0x404); + } + if a.reduce_min() != *a_arr.iter().min().unwrap() || a.reduce_max() != *a_arr.iter().max().unwrap() { + return Err(0x405); + } + // `abs`/`neg` wrap at i32::MIN on every backend (release semantics). + if a.abs().to_array() != core::array::from_fn(|i| a_arr[i].wrapping_abs()) { + return Err(0x406); + } + if (-a).to_array() != core::array::from_fn(|i| a_arr[i].wrapping_neg()) { + return Err(0x407); + } + if (a + b).to_array() != core::array::from_fn(|i| a_arr[i].wrapping_add(b_arr[i])) { + return Err(0x408); + } + if (a * b).to_array() != core::array::from_fn(|i| a_arr[i].wrapping_mul(b_arr[i])) { + return Err(0x409); + } + let mut sum = 0i32; + for &x in &a_arr { + sum = sum.wrapping_add(x); + } + if a.reduce_sum() != sum { + return Err(0x40A); + } + let i16s: [i16; 16] = core::array::from_fn(|i| (i as i16 - 8) * 1000); + if I32x16::from_i16_slice(&i16s).to_array() != core::array::from_fn(|i| i16s[i] as i32) { + return Err(0x40B); + } + if a.to_i16_array() != core::array::from_fn(|i| a_arr[i] as i16) { + return Err(0x40C); + } + if !(a == I32x16::from_array(a_arr)) || a == b { + return Err(0x40D); + } + Ok(()) + } } diff --git a/crates/simd-masking-parity/Cargo.toml b/crates/simd-masking-parity/Cargo.toml new file mode 100644 index 00000000..be6d1cb0 --- /dev/null +++ b/crates/simd-masking-parity/Cargo.toml @@ -0,0 +1,39 @@ +# simd-masking-parity — ONE masking parity program, compiled under every SIMD +# realization the matrix certifies (scalar / v3-avx2 / v4-avx512 / native / +# nightly) × every platform (linux-x86 / linux-arm64 / macos-arm64 / +# wasm32-simd128 / wasm32-without-simd128 = the scalar arm). +# +# It calls the SHIPPED `ndarray::simd` facade only — never a backend module, +# never an intrinsic, never a `cfg(target_feature)` on lane data — and checks +# every result against a bit-serial reference computed in the same module. +# Whatever backend `simd.rs` selected for the build is the thing under test; +# the program has no idea which one that is, and must not. +# +# Ships as an rlib + cdylib (`selfcheck()` for the node harness on wasm) and a +# bin (`main.rs`, exits non-zero on the first mismatch, for native / qemu). +# EXCLUDED from the workspace (root Cargo.toml `exclude`) so it has zero effect +# on ordinary builds; `scripts/masking-parity.sh` builds it via --manifest-path. +[package] +name = "simd-masking-parity" +version = "0.0.0" +edition = "2021" +publish = false + +[lib] +crate-type = ["cdylib", "rlib"] + +[[bin]] +name = "simd-masking-parity" +path = "src/main.rs" + +[dependencies] +# `simd` + `simd_masking_ops` need only ndarray's `std` feature. +ndarray = { path = "../..", default-features = false, features = ["std"] } + +[features] +# The nightly realization: forwards to ndarray's portable-simd backend. +nightly-simd = ["ndarray/nightly-simd"] + +[profile.release] +panic = "abort" +opt-level = 2 diff --git a/crates/simd-masking-parity/run.mjs b/crates/simd-masking-parity/run.mjs new file mode 100644 index 00000000..e51dc963 --- /dev/null +++ b/crates/simd-masking-parity/run.mjs @@ -0,0 +1,18 @@ +// Node harness for the wasm arms of the masking parity matrix: instantiate the +// .wasm and assert selfcheck() == 0. A nonzero rc names the failing group+op +// (see src/lib.rs, `Code`). +import { readFileSync } from 'node:fs'; +const wasmPath = process.argv[2]; +if (!wasmPath) { + console.error('usage: node run.mjs '); + process.exit(2); +} +const { instance } = await WebAssembly.instantiate(readFileSync(wasmPath), {}); +const rc = instance.exports.selfcheck(); +if (rc === 0) { + console.log('masking parity: OK (selfcheck rc=0)'); + process.exit(0); +} else { + console.error(`masking parity: FAIL — selfcheck rc=0x${rc.toString(16)} (see crates/simd-masking-parity/src/lib.rs)`); + process.exit(1); +} diff --git a/crates/simd-masking-parity/src/lib.rs b/crates/simd-masking-parity/src/lib.rs new file mode 100644 index 00000000..da1c24aa --- /dev/null +++ b/crates/simd-masking-parity/src/lib.rs @@ -0,0 +1,759 @@ +//! The ONE masking parity program of the SIMD realization matrix. +//! +//! [`run`] exercises the shipped `ndarray::simd` masking facade — every +//! predicate→mask, the mask algebra, all 256 `ternlog` tables on both mask +//! lane widths, the care-masked register matches (contiguous and strided), +//! the tail rules at 0 / 1 / 63 / 64 / 65 / 130 rows, the masked reductions, +//! and `blend` — against references computed here (bit-serial for `ternlog`, +//! scalar word-serial for the rest; none of them touches an `ndarray::simd` +//! type), and returns `0` iff every result is bit-identical. It has NO idea which backend +//! `simd.rs` selected: no `cfg(target_feature)`, no backend module, no +//! intrinsic. The build (its `-Ctarget-cpu`, its target, its +//! `nightly-simd` feature) chooses the realization; this program only asks +//! whether that realization agrees with the definition. +//! +//! Return codes are distinct per assertion so a CI failure names the exact +//! op and shape: `0x1IM` / `0x2IM` = ternlog table `IM` on `U32x16` / +//! `U64x8`, `0x3xx` `U64x8` algebra, `0x4xx` `I32x16` compares, `0x5xx` +//! predicate→mask, `0x6xx` mask algebra, `0x7xx` care-match, `0x8xx` masked +//! reductions and blend. `main.rs` (native / qemu) and `selfcheck()` (the +//! wasm cdylib export, driven by `run.mjs`) both call [`run`]. + +use ndarray::simd::{ + blend_i32, eq_i32_to_mask, eq_u32_strided_to_mask, eq_u32_to_mask, ge_i32_to_mask, gt_i32_to_mask, le_i32_to_mask, + lt_i32_to_mask, mask_all, mask_and, mask_and_assign, mask_andnot, mask_andnot_assign, mask_any, mask_not, + mask_not_assign, mask_or, mask_or_assign, mask_ternlog, mask_ternlog_assign, mask_xor, mask_xor_assign, + masked_max_i32, masked_min_i32, masked_strided_group_sum, masked_sum_i32, ne_i32_to_mask, ne_u32_to_mask, + ternary_match_strided_to_mask, ternary_match_u32_to_mask, ternary_match_u64_to_mask, ternlog, I32x16, U32x16, + U64x8, +}; + +/// Number of check groups [`run`] executes (for the log line only). +pub const CHECKS: usize = 8; + +/// The wasm export: identical to [`run`], `extern "C"` so `run.mjs` can call it. +#[no_mangle] +pub extern "C" fn selfcheck() -> u32 { + run() +} + +/// Run every group; `0` on success, else the code of the first failing assertion. +pub fn run() -> u32 { + let groups: [fn() -> Result<(), u32>; CHECKS] = [ + check_ternlog_all_tables, check_u64x8_algebra, check_i32x16_compare, check_predicates_to_mask, + check_mask_algebra, check_care_match, check_masked_reductions, check_blend, + ]; + for g in groups { + if let Err(code) = g() { + return code; + } + } + 0 +} + +/// The row lengths every slice-level check runs at: the empty mask, a single +/// row, one word minus one, exactly one word, one word plus one, and a +/// three-word mask with a partial tail. +const LENS: [usize; 6] = [0, 1, 63, 64, 65, 130]; + +/// Deterministic SplitMix64 so every realization sees the same operands. +struct SplitMix64(u64); +impl SplitMix64 { + fn next(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } +} + +fn words_for(n: usize) -> usize { + n.div_ceil(64) +} + +/// Bit-serial ternlog reference: bit `i` of the result is `imm >> (a_i<<2 | b_i<<1 | c_i)`. +fn reference_ternlog_u64(imm: u32, a: u64, b: u64, c: u64) -> u64 { + let mut r = 0u64; + for bit in 0..64 { + let idx = (((a >> bit) & 1) << 2) | (((b >> bit) & 1) << 1) | ((c >> bit) & 1); + r |= (((imm >> idx) & 1) as u64) << bit; + } + r +} + +/// The reference mask writer every predicate is checked against: bit `i` of +/// word `i / 64` iff `pred(i)`, every other bit (tail and surplus words) zero. +fn reference_mask(n: usize, out_len: usize, pred: impl Fn(usize) -> bool) -> Vec { + let mut w = vec![0u64; out_len]; + for i in 0..n { + if pred(i) { + w[i / 64] |= 1u64 << (i % 64); + } + } + w +} + +/// Mixed-sign `i32` operands with the extremes and equal pairs at every length. +fn i32_values(n: usize, rng: &mut SplitMix64) -> Vec { + let fixed = [i32::MIN, i32::MAX, 0, -1, 1, 7, 7, -7, 100, -100]; + (0..n) + .map(|i| if i < fixed.len() { fixed[i] } else { rng.next() as i32 }) + .collect() +} + +fn u32_values(n: usize, rng: &mut SplitMix64) -> Vec { + let fixed = [0, u32::MAX, 1, 0x8000_0000, 7, 7, 0xDEAD_BEEF]; + (0..n) + .map(|i| if i < fixed.len() { fixed[i] } else { rng.next() as u32 }) + .collect() +} + +// ── 0x1xx / 0x2xx: ternlog, all 256 tables, both lane widths ──────────────── + +fn check_ternlog_all_tables() -> Result<(), u32> { + let u32_triples: [([u32; 16], [u32; 16], [u32; 16]); 2] = [ + ([0xF0F0_F0F0; 16], [0xCCCC_CCCC; 16], [0xAAAA_AAAA; 16]), + ( + [ + 0x0000_0000, 0xFFFF_FFFF, 0x0000_0001, 0x8000_0000, 0x1234_5678, 0x9ABC_DEF0, 0xDEAD_BEEF, 0xCAFE_BABE, + 0x0F0F_0F0F, 0xF0F0_F0F0, 0x5555_5555, 0xAAAA_AAAA, 0x0000_00FF, 0xFF00_0000, 0x0101_0101, 0x8080_8080, + ], + [ + 0x9E37_79B9, 0x1111_1111, 0xDEAD_C0DE, 0x0BAD_F00D, 0x7FFF_FFFF, 0x0000_0000, 0xFFFF_FFFF, 0x1357_9BDF, + 0x2468_ACE0, 0xFEDC_BA98, 0x0000_0010, 0x0000_001F, 0xABCD_EF01, 0x1020_4080, 0x0F0F_F0F0, 0xC0DE_CAFE, + ], + [ + 0x1357_9BDF, 0xC0DE_CAFE, 0x0000_0000, 0xFFFF_FFFF, 0xA5A5_A5A5, 0x5A5A_5A5A, 0x0000_8000, 0x8000_0001, + 0x7777_7777, 0x8888_8888, 0x0F0F_0F0F, 0xF0F0_F0F0, 0x1234_5678, 0x8765_4321, 0xFFFF_0000, 0x0000_FFFF, + ], + ), + ]; + let u64_triples: [([u64; 8], [u64; 8], [u64; 8]); 2] = [ + ([0xF0F0_F0F0_F0F0_F0F0; 8], [0xCCCC_CCCC_CCCC_CCCC; 8], [0xAAAA_AAAA_AAAA_AAAA; 8]), + ( + [ + 0, + u64::MAX, + 1, + 1 << 63, + 0x1234_5678_9ABC_DEF0, + 0xDEAD_BEEF_CAFE_BABE, + 0x0F0F_0F0F_F0F0_F0F0, + 0x5555_AAAA_5555_AAAA, + ], + [ + 0x9E37_79B9_7F4A_7C15, + 0x1111_1111_1111_1111, + 0xDEAD_C0DE_0BAD_F00D, + u64::MAX, + 0, + 0x1357_9BDF_2468_ACE0, + 0xFEDC_BA98_7654_3210, + 0xC0DE_CAFE_C0DE_CAFE, + ], + [ + 0x1357_9BDF_1357_9BDF, + 0, + u64::MAX, + 0xA5A5_A5A5_5A5A_5A5A, + 0x8000_0000_0000_0001, + 0x7777_7777_8888_8888, + 0xFFFF_0000_0000_FFFF, + 0x0123_4567_89AB_CDEF, + ], + ), + ]; + + macro_rules! check_imm { + ($imm:expr) => {{ + const IMM: i32 = $imm; + for (a, b, c) in &u32_triples { + let got = U32x16::from_array(*a) + .ternlog::(U32x16::from_array(*b), U32x16::from_array(*c)) + .to_array(); + for i in 0..16 { + let want = reference_ternlog_u64(IMM as u32, a[i] as u64, b[i] as u64, c[i] as u64) as u32; + if got[i] != want { + return Err(0x100 | IMM as u32); + } + } + } + for (a, b, c) in &u64_triples { + let got = U64x8::from_array(*a) + .ternlog::(U64x8::from_array(*b), U64x8::from_array(*c)) + .to_array(); + for i in 0..8 { + if got[i] != reference_ternlog_u64(IMM as u32, a[i], b[i], c[i]) { + return Err(0x200 | IMM as u32); + } + } + } + }}; + } + macro_rules! check_row { + ($hi:expr) => { + check_imm!($hi * 16 + 0); + check_imm!($hi * 16 + 1); + check_imm!($hi * 16 + 2); + check_imm!($hi * 16 + 3); + check_imm!($hi * 16 + 4); + check_imm!($hi * 16 + 5); + check_imm!($hi * 16 + 6); + check_imm!($hi * 16 + 7); + check_imm!($hi * 16 + 8); + check_imm!($hi * 16 + 9); + check_imm!($hi * 16 + 10); + check_imm!($hi * 16 + 11); + check_imm!($hi * 16 + 12); + check_imm!($hi * 16 + 13); + check_imm!($hi * 16 + 14); + check_imm!($hi * 16 + 15); + }; + } + check_row!(0); + check_row!(1); + check_row!(2); + check_row!(3); + check_row!(4); + check_row!(5); + check_row!(6); + check_row!(7); + check_row!(8); + check_row!(9); + check_row!(10); + check_row!(11); + check_row!(12); + check_row!(13); + check_row!(14); + check_row!(15); + Ok(()) +} + +// ── 0x3xx: U64x8 algebra (the operators the bulk mask ops ride) ───────────── + +fn check_u64x8_algebra() -> Result<(), u32> { + let a_arr: [u64; 8] = [ + 0, + u64::MAX, + 1 << 63, + 0xF0F0_F0F0_F0F0_F0F0, + 0x1234_5678_9ABC_DEF0, + 1, + 0xDEAD_BEEF_CAFE_BABE, + 0x5555_AAAA_5555_AAAA, + ]; + let b_arr: [u64; 8] = [ + u64::MAX, + 0, + 1 << 63, + 0xCCCC_CCCC_CCCC_CCCC, + 0x0FED_CBA9_8765_4321, + 1, + 0xC0DE_CAFE_C0DE_CAFE, + 0xAAAA_5555_AAAA_5555, + ]; + let (a, b) = (U64x8::from_array(a_arr), U64x8::from_array(b_arr)); + if a.to_array() != a_arr { + return Err(0x300); + } + let mut back = [0u64; 8]; + U64x8::from_slice(&a_arr).copy_to_slice(&mut back); + if back != a_arr { + return Err(0x301); + } + let (and, or, xor, not, andnot) = + ((a & b).to_array(), (a | b).to_array(), (a ^ b).to_array(), (!a).to_array(), a.andnot(b).to_array()); + let pop = a.popcnt().to_array(); + let mut hamming = 0u64; + for i in 0..8 { + if and[i] != a_arr[i] & b_arr[i] { + return Err(0x302); + } + if or[i] != a_arr[i] | b_arr[i] { + return Err(0x303); + } + if xor[i] != a_arr[i] ^ b_arr[i] { + return Err(0x304); + } + if not[i] != !a_arr[i] { + return Err(0x305); + } + if andnot[i] != a_arr[i] & !b_arr[i] { + return Err(0x306); + } + if pop[i] != a_arr[i].count_ones() as u64 { + return Err(0x307); + } + hamming += (a_arr[i] ^ b_arr[i]).count_ones() as u64; + } + if a.xor_popcount(b) != hamming { + return Err(0x308); + } + for &n in &[0u32, 1, 7, 31, 32, 63] { + let (l, r) = (a.rotate_left(n).to_array(), a.rotate_right(n).to_array()); + for i in 0..8 { + if l[i] != a_arr[i].rotate_left(n) { + return Err(0x309); + } + if r[i] != a_arr[i].rotate_right(n) { + return Err(0x30A); + } + } + } + let mut sum = 0u64; + for &x in &a_arr { + sum = sum.wrapping_add(x); + } + if a.reduce_sum() != sum { + return Err(0x30B); + } + if (a + b).to_array() != core::array::from_fn(|i| a_arr[i].wrapping_add(b_arr[i])) { + return Err(0x30C); + } + if (a - b).to_array() != core::array::from_fn(|i| a_arr[i].wrapping_sub(b_arr[i])) { + return Err(0x30D); + } + if !(a == U64x8::from_array(a_arr)) || a == b { + return Err(0x30E); + } + Ok(()) +} + +// ── 0x4xx: I32x16 compares and reductions (the ordered-predicate primitive) ─ + +fn check_i32x16_compare() -> Result<(), u32> { + let a_arr: [i32; 16] = + [i32::MIN, i32::MAX, 0, -1, 1, -2, 7, 7, 100, -100, 0x7FFF_0000, -0x7FFF_0000, 42, -42, 2, -3]; + let b_arr: [i32; 16] = + [i32::MIN, i32::MAX, 0, 0, -1, -2, 7, 8, -100, 100, -0x7FFF_0000, 0x7FFF_0000, 41, -41, -3, 2]; + let (a, b) = (I32x16::from_array(a_arr), I32x16::from_array(b_arr)); + if a.to_array() != a_arr { + return Err(0x400); + } + let gt = a.gt_bitmask(b); + let ge0 = a.cmpge_zero_mask(); + for i in 0..16 { + if ((gt >> i) & 1 == 1) != (a_arr[i] > b_arr[i]) { + return Err(0x401); + } + if ((ge0 >> i) & 1 == 1) != (a_arr[i] >= 0) { + return Err(0x402); + } + } + if a.simd_min(b).to_array() != core::array::from_fn(|i| a_arr[i].min(b_arr[i])) { + return Err(0x403); + } + if a.simd_max(b).to_array() != core::array::from_fn(|i| a_arr[i].max(b_arr[i])) { + return Err(0x404); + } + if a.reduce_min() != i32::MIN || a.reduce_max() != i32::MAX { + return Err(0x405); + } + // Extremes in EVERY lane position, so a reduction tree that drops a lane + // (the 16→8→4→2→1 ladders) is caught wherever the drop happens. + for lane in 0..16 { + let mut arr = [0i32; 16]; + arr[lane] = i32::MIN; + arr[(lane + 5) % 16] = i32::MAX; + let v = I32x16::from_array(arr); + if v.reduce_min() != i32::MIN || v.reduce_max() != i32::MAX { + return Err(0x406); + } + } + let mut sum = 0i32; + for &x in &a_arr { + sum = sum.wrapping_add(x); + } + if a.reduce_sum() != sum { + return Err(0x407); + } + if !(a == I32x16::from_array(a_arr)) || a == b { + return Err(0x408); + } + Ok(()) +} + +// ── 0x5xx: predicate → mask, every tail shape, full-overwrite + zero tail ──── + +fn check_predicates_to_mask() -> Result<(), u32> { + let mut rng = SplitMix64(0x5EED_0000_0001); + for &n in &LENS { + // One surplus word beyond ceil(n/64) so the "surplus words are written + // zero" half of the contract is observable, pre-filled with garbage so + // a writer that ORs instead of overwriting is caught. + let out_len = words_for(n) + 1; + let vals = i32_values(n, &mut rng); + let uvals = u32_values(n, &mut rng); + let thresholds = [i32::MIN, -1, 0, 1, 7, i32::MAX]; + let mut out = vec![u64::MAX; out_len]; + for (k, &t) in thresholds.iter().enumerate() { + let k = k as u32; + macro_rules! pred { + ($f:ident, $op:tt, $code:expr) => {{ + out.iter_mut().for_each(|w| *w = u64::MAX); + $f(&vals, t, &mut out); + if out != reference_mask(n, out_len, |i| vals[i] $op t) { + return Err($code | k); + } + }}; + } + pred!(eq_i32_to_mask, ==, 0x500); + pred!(ne_i32_to_mask, !=, 0x510); + pred!(lt_i32_to_mask, <, 0x520); + pred!(le_i32_to_mask, <=, 0x530); + pred!(gt_i32_to_mask, >, 0x540); + pred!(ge_i32_to_mask, >=, 0x550); + } + for (k, &needle) in [0u32, 7, u32::MAX, 0x8000_0000].iter().enumerate() { + let k = k as u32; + out.iter_mut().for_each(|w| *w = u64::MAX); + eq_u32_to_mask(&uvals, needle, &mut out); + if out != reference_mask(n, out_len, |i| uvals[i] == needle) { + return Err(0x560 | k); + } + out.iter_mut().for_each(|w| *w = u64::MAX); + ne_u32_to_mask(&uvals, needle, &mut out); + if out != reference_mask(n, out_len, |i| uvals[i] != needle) { + return Err(0x570 | k); + } + } + // Strided: a u32 at byte 4 of every 16-byte record. + let mut bytes = vec![0u8; n * 16 + 3]; + for i in 0..n { + bytes[4 + i * 16..8 + i * 16].copy_from_slice(&uvals[i].to_le_bytes()); + } + out.iter_mut().for_each(|w| *w = u64::MAX); + eq_u32_strided_to_mask(&bytes, 4, 16, n, 7, &mut out); + if out != reference_mask(n, out_len, |i| uvals[i] == 7) { + return Err(0x580); + } + } + Ok(()) +} + +// ── 0x6xx: mask algebra, in-place forms, ternlog over slices, any / all ───── + +/// Word counts for the mask-algebra group. `LENS` are ROW counts, and at 130 +/// rows a mask is only 3 words — below one `U64x8` chunk — so iterating `LENS` +/// here would run the padded tail of every word op and never its `as_chunks` +/// body (savant-architect, PR #306). These counts straddle the 8-word chunk +/// boundary and cover every tail length 1..=7. +const WORD_LENS: [usize; 14] = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 15, 16, 17, 31]; + +/// Row counts for the mask-algebra group: every full-word count in +/// `WORD_LENS`, plus, for each non-zero word count, three partially-live +/// final words (63, 32 and 1 live bits). The full-word counts keep the +/// `as_chunks` bodies covered; the partial ones are the ONLY inputs that +/// reach the tail branches of `mask_not` / `mask_not_assign` / `mask_all` and +/// the tail-only `mask_any` check below — with `n` always `nw * 64`, that +/// branch was dead and a backend mishandling tail bits would still have +/// passed (CodeRabbit on PR #306). +fn row_lens() -> Vec { + let mut v = Vec::with_capacity(WORD_LENS.len() * 4); + for &nw in &WORD_LENS { + v.push(nw * 64); + if nw > 0 { + v.push(nw * 64 - 1); + v.push(nw * 64 - 32); + v.push(nw * 64 - 63); + } + } + v +} + +fn check_mask_algebra() -> Result<(), u32> { + let mut rng = SplitMix64(0x6000_0000_0002); + for n in row_lens() { + let nw = words_for(n); + // Conforming inputs: random bits with the tail beyond `n` cleared. + let tail_mask = |i: usize| -> u64 { + let live = n.saturating_sub(i * 64).min(64); + if live == 64 { + u64::MAX + } else { + (1u64 << live) - 1 + } + }; + let a: Vec = (0..nw).map(|i| rng.next() & tail_mask(i)).collect(); + let b: Vec = (0..nw).map(|i| rng.next() & tail_mask(i)).collect(); + let c: Vec = (0..nw).map(|i| rng.next() & tail_mask(i)).collect(); + let mut dst = vec![u64::MAX; nw]; + + mask_and(&a, &b, &mut dst); + if dst.iter().zip(&a).zip(&b).any(|((&d, &x), &y)| d != x & y) { + return Err(0x600); + } + mask_or(&a, &b, &mut dst); + if dst.iter().zip(&a).zip(&b).any(|((&d, &x), &y)| d != x | y) { + return Err(0x601); + } + mask_xor(&a, &b, &mut dst); + if dst.iter().zip(&a).zip(&b).any(|((&d, &x), &y)| d != x ^ y) { + return Err(0x602); + } + mask_andnot(&a, &b, &mut dst); + if dst.iter().zip(&a).zip(&b).any(|((&d, &x), &y)| d != x & !y) { + return Err(0x603); + } + mask_not(&a, n, &mut dst); + if dst + .iter() + .enumerate() + .any(|(i, &d)| d != !a[i] & tail_mask(i)) + { + return Err(0x604); + } + // In-place forms agree with the out-of-place ones. + let mut t = a.clone(); + mask_and_assign(&mut t, &b); + if t.iter().zip(&a).zip(&b).any(|((&d, &x), &y)| d != x & y) { + return Err(0x605); + } + let mut t = a.clone(); + mask_or_assign(&mut t, &b); + if t.iter().zip(&a).zip(&b).any(|((&d, &x), &y)| d != x | y) { + return Err(0x606); + } + let mut t = a.clone(); + mask_xor_assign(&mut t, &b); + if t.iter().zip(&a).zip(&b).any(|((&d, &x), &y)| d != x ^ y) { + return Err(0x607); + } + let mut t = a.clone(); + mask_andnot_assign(&mut t, &b); + if t.iter().zip(&a).zip(&b).any(|((&d, &x), &y)| d != x & !y) { + return Err(0x608); + } + let mut t = a.clone(); + mask_not_assign(&mut t, n); + if t.iter() + .enumerate() + .any(|(i, &d)| d != !a[i] & tail_mask(i)) + { + return Err(0x609); + } + // Slice-level ternlog at the named immediates plus the two constant + // tables and the odd-parity one. + macro_rules! slice_ternlog { + ($imm:expr, $code:expr) => {{ + const IMM: i32 = $imm; + mask_ternlog::(&a, &b, &c, &mut dst); + for i in 0..nw { + if dst[i] != reference_ternlog_u64(IMM as u32, a[i], b[i], c[i]) { + return Err($code); + } + } + let mut t = a.clone(); + mask_ternlog_assign::(&mut t, &b, &c); + if t != dst { + return Err($code | 0x8); + } + }}; + } + slice_ternlog!(ternlog::AND2_OR, 0x610); + slice_ternlog!(ternlog::XOR_AND, 0x611); + slice_ternlog!(ternlog::MAJ3, 0x612); + slice_ternlog!(ternlog::AND3, 0x613); + slice_ternlog!(ternlog::OR3, 0x614); + slice_ternlog!(0x00, 0x615); + slice_ternlog!(0xFF, 0x616); + slice_ternlog!(0x96, 0x617); + + // any / all on the tail shapes: all-zero, all-live-set, one bit + // missing (the LAST live row), and a set bit ONLY in the tail (a + // non-conforming mask, which `any` must still see and `all` ignore). + let zero = vec![0u64; nw]; + let full: Vec = (0..nw).map(tail_mask).collect(); + if mask_any(&zero) { + return Err(0x620); + } + if mask_any(&full) != (n > 0) { + return Err(0x621); + } + if !mask_all(&zero, 0) || !mask_all(&full, n) { + return Err(0x622); + } + if n > 0 { + if mask_all(&zero, n) { + return Err(0x623); + } + let mut missing = full.clone(); + missing[(n - 1) / 64] &= !(1u64 << ((n - 1) % 64)); + if mask_all(&missing, n) || (n > 1 && !mask_any(&missing)) { + return Err(0x624); + } + if n % 64 != 0 { + let mut tail_only = zero.clone(); + tail_only[nw - 1] = 1u64 << (n % 64); + if !mask_any(&tail_only) || mask_all(&tail_only, n) { + return Err(0x625); + } + } + } + } + Ok(()) +} + +// ── 0x7xx: care-masked matches — contiguous u32 / u64 and the strided 12-byte register ── + +fn check_care_match() -> Result<(), u32> { + let mut rng = SplitMix64(0x7000_0000_0003); + let cases_u32: [(u32, u32); 5] = [ + (0, 0), + (7, u32::MAX), + (0xDEAD_BEEF, 0xFFFF_0000), + (0x8000_0000, 0x8000_0000), + (0x1234_5678, 0x0F0F_0F0F), + ]; + for &n in &LENS { + let out_len = words_for(n) + 1; + let vals = u32_values(n, &mut rng); + let vals64: Vec = (0..n) + .map(|i| if i % 3 == 0 { vals[i] as u64 } else { rng.next() }) + .collect(); + let mut out = vec![u64::MAX; out_len]; + for (k, &(pattern, care)) in cases_u32.iter().enumerate() { + out.iter_mut().for_each(|w| *w = u64::MAX); + ternary_match_u32_to_mask(&vals, pattern, care, &mut out); + if out != reference_mask(n, out_len, |i| (vals[i] ^ pattern) & care == 0) { + return Err(0x700 | k as u32); + } + let (p64, c64) = ((pattern as u64) << 32 | pattern as u64, (care as u64) << 32 | care as u64); + out.iter_mut().for_each(|w| *w = u64::MAX); + ternary_match_u64_to_mask(&vals64, p64, c64, &mut out); + if out != reference_mask(n, out_len, |i| (vals64[i] ^ p64) & c64 == 0) { + return Err(0x710 | k as u32); + } + } + // Strided 12-byte register at byte 4 of each 16-byte facet; every + // third record is forced to match on the cared bytes. + let mut bytes = vec![0u8; n * 16 + 5]; + let pattern: [u8; 12] = [0xA5, 0x00, 0x5A, 0xFF, 0x01, 0x02, 0x03, 0x04, 0x80, 0x7F, 0x10, 0x20]; + let care: [u8; 12] = [0xFF, 0x00, 0x0F, 0xFF, 0xFF, 0x00, 0xFF, 0x00, 0x80, 0x7F, 0xFF, 0xFF]; + for i in 0..n { + let reg = &mut bytes[4 + i * 16..16 + i * 16]; + for (k, b) in reg.iter_mut().enumerate() { + *b = if i % 3 == 0 { + pattern[k] ^ (!care[k] & (rng.next() as u8)) + } else { + rng.next() as u8 + }; + } + } + out.iter_mut().for_each(|w| *w = u64::MAX); + ternary_match_strided_to_mask(&bytes, 4, 16, n, &pattern, &care, &mut out); + let want = reference_mask(n, out_len, |i| { + let reg = &bytes[4 + i * 16..16 + i * 16]; + (0..12).all(|k| (reg[k] ^ pattern[k]) & care[k] == 0) + }); + if out != want { + return Err(0x720); + } + // Anti-vacuity: the forced records must actually match, and at least + // one unforced record must not (for n >= 2), or the check proved nothing. + if n >= 2 && (want[0] & 1 == 0 || want.iter().map(|w| w.count_ones()).sum::() as usize == n) { + return Err(0x721); + } + } + Ok(()) +} + +// ── 0x8xx: masked reductions ──────────────────────────────────────────────── + +fn check_masked_reductions() -> Result<(), u32> { + let mut rng = SplitMix64(0x8000_0000_0004); + for &n in &LENS { + let nw = words_for(n); + let vals = i32_values(n, &mut rng); + let tail_mask = |i: usize| -> u64 { + let live = n.saturating_sub(i * 64).min(64); + if live == 64 { + u64::MAX + } else { + (1u64 << live) - 1 + } + }; + let masks: [Vec; 4] = [ + vec![0u64; nw], + (0..nw).map(tail_mask).collect(), + (0..nw).map(|i| rng.next() & tail_mask(i)).collect(), + (0..nw) + .map(|i| 0x8000_0000_0000_0001u64 & tail_mask(i)) + .collect(), + ]; + for (k, m) in masks.iter().enumerate() { + let k = k as u32; + let selected: Vec = (0..n) + .filter(|&i| (m[i / 64] >> (i % 64)) & 1 == 1) + .map(|i| vals[i]) + .collect(); + let want_sum: i64 = selected.iter().map(|&v| v as i64).sum(); + if masked_sum_i32(&vals, m) != want_sum { + return Err(0x800 | k); + } + if masked_min_i32(&vals, m) != selected.iter().copied().min() { + return Err(0x810 | k); + } + if masked_max_i32(&vals, m) != selected.iter().copied().max() { + return Err(0x820 | k); + } + } + // The extremes must survive the masked min/max exactly, not saturate. + if n > 0 { + let full: Vec = (0..nw).map(tail_mask).collect(); + if masked_min_i32(&vals, &full) != Some(i32::MIN) + || (n > 1 && masked_max_i32(&vals, &full) != Some(i32::MAX)) + { + return Err(0x830); + } + } + // Strided group sum: 16-byte records, a register of `groups` fields of + // `group_bytes` bytes at offset 4, for all three legal widths. + for (gk, &(groups, group_bytes)) in [(3usize, 2usize), (6, 1), (2, 4), (1, 4)] + .iter() + .enumerate() + { + let mut bytes = vec![0u8; n * 16 + 1]; + let mut fields: Vec> = Vec::with_capacity(n); + for i in 0..n { + let mut rec = Vec::with_capacity(groups); + for g in 0..groups { + let v = rng.next() & ((1u64 << (8 * group_bytes)) - 1); + let off = 4 + i * 16 + g * group_bytes; + bytes[off..off + group_bytes].copy_from_slice(&v.to_le_bytes()[..group_bytes]); + rec.push(v); + } + fields.push(rec); + } + let m = &masks[2]; + let want: i64 = (0..n) + .filter(|&i| (m[i / 64] >> (i % 64)) & 1 == 1) + .map(|i| fields[i].iter().sum::() as i64) + .sum(); + if masked_strided_group_sum(&bytes, 4, 16, n, groups, group_bytes, m) != Some(want) { + return Err(0x840 | gk as u32); + } + } + } + Ok(()) +} + +fn check_blend() -> Result<(), u32> { + let mut rng = SplitMix64(0x8800_0000_0005); + for &n in &LENS { + let nw = words_for(n); + let a = i32_values(n, &mut rng); + let b: Vec = (0..n).map(|_| rng.next() as i32).collect(); + let m: Vec = (0..nw).map(|_| rng.next()).collect(); + let mut dst = vec![0i32; n]; + blend_i32(&m, &a, &b, &mut dst); + for i in 0..n { + let want = if (m[i / 64] >> (i % 64)) & 1 == 1 { a[i] } else { b[i] }; + if dst[i] != want { + return Err(0x880); + } + } + } + Ok(()) +} diff --git a/crates/simd-masking-parity/src/main.rs b/crates/simd-masking-parity/src/main.rs new file mode 100644 index 00000000..ca7fabb7 --- /dev/null +++ b/crates/simd-masking-parity/src/main.rs @@ -0,0 +1,25 @@ +//! Native / qemu entry: run the shared masking parity program and exit +//! non-zero on the first mismatch. The cfg lines it prints are a LOG of which +//! realization this binary was built as — the checks themselves never branch +//! on them (the whole point is that the same program runs under every arm). + +fn main() { + println!( + "simd-masking-parity: arch={} avx2={} avx512f={} neon={} simd128={} nightly-simd={}", + std::env::consts::ARCH, + cfg!(target_feature = "avx2"), + cfg!(target_feature = "avx512f"), + cfg!(target_feature = "neon"), + cfg!(target_feature = "simd128"), + cfg!(feature = "nightly-simd"), + ); + let rc = simd_masking_parity::run(); + if rc != 0 { + eprintln!("simd-masking-parity FAILED: code = {rc:#x} (see src/lib.rs)"); + std::process::exit(1); + } + println!( + "simd-masking-parity OK: {} check groups bit-identical to their scalar / bit-serial references", + simd_masking_parity::CHECKS + ); +} diff --git a/crates/wasm-simd-parity/src/lib.rs b/crates/wasm-simd-parity/src/lib.rs index c333a6ec..b84c4a6c 100644 --- a/crates/wasm-simd-parity/src/lib.rs +++ b/crates/wasm-simd-parity/src/lib.rs @@ -26,6 +26,15 @@ pub extern "C" fn selfcheck() -> u32 { if let Err(code) = check_i8x16() { return code; } + if let Err(code) = check_ternlog_all_tables() { + return code; + } + if let Err(code) = check_u64x8_algebra() { + return code; + } + if let Err(code) = check_i32x16_compare() { + return code; + } 0 } @@ -69,9 +78,7 @@ fn check_u32x16() -> Result<(), u32> { /// `F32x16` — the float hot-path lane (splat / roundtrip / add / reduce_sum). fn check_f32x16() -> Result<(), u32> { - let data: [f32; 16] = [ - 0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, - ]; + let data: [f32; 16] = [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0]; if F32x16::from_array(data).to_array() != data { return Err(20); } @@ -97,7 +104,9 @@ fn check_i8x16() -> Result<(), u32> { if I8x16::from_array(a_arr).to_array() != a_arr { return Err(30); } - let sum = I8x16::from_array(a_arr).add(I8x16::from_array(b_arr)).to_array(); + let sum = I8x16::from_array(a_arr) + .add(I8x16::from_array(b_arr)) + .to_array(); for i in 0..16 { if sum[i] != a_arr[i].wrapping_add(b_arr[i]) { return Err(31); @@ -105,3 +114,313 @@ fn check_i8x16() -> Result<(), u32> { } Ok(()) } + +/// `ternlog::` over ALL 256 truth tables, on BOTH lane types this target +/// resolves through `ndarray::simd`: the native `U32x16` and the native +/// `U64x8` (both `[v128; 4]` fan-outs with generated backend-local bodies — +/// `U64x8` was the scalar backend's re-export until the 2026-09-13 +/// five-flavour audit). The reference is bit-serial and lives here, +/// independent of every backend. This proves the WASM arm only: the x86 arms +/// run under `cargo test` (`simd::tests::w1a9_*`), NEON under +/// `crates/neon-simd-parity` on qemu. +/// +/// Operands: the canonical `F0/CC/AA` triple (every one of the 8 index +/// combinations appears in every byte) plus a second triple with mixed lane +/// values so a per-lane transposition would be caught as well. Return codes: +/// `40 + ` nothing lane-specific — the failing table is reported through the +/// code `0x100 | IMM` for `U32x16` and `0x200 | IMM` for `U64x8`, so a CI +/// failure names the exact truth table. +fn check_ternlog_all_tables() -> Result<(), u32> { + use ndarray::simd::U64x8; + + fn reference_u64(imm: u32, a: u64, b: u64, c: u64) -> u64 { + let mut r = 0u64; + for bit in 0..64 { + let idx = (((a >> bit) & 1) << 2) | (((b >> bit) & 1) << 1) | ((c >> bit) & 1); + r |= (((imm >> idx) & 1) as u64) << bit; + } + r + } + + // Two operand triples per lane width. Triple 0 is the canonical + // all-8-minterms pattern; triple 1 mixes per-lane values. + let u32_triples: [([u32; 16], [u32; 16], [u32; 16]); 2] = [ + ([0xF0F0_F0F0; 16], [0xCCCC_CCCC; 16], [0xAAAA_AAAA; 16]), + ( + [ + 0x0000_0000, 0xFFFF_FFFF, 0x0000_0001, 0x8000_0000, 0x1234_5678, 0x9ABC_DEF0, 0xDEAD_BEEF, 0xCAFE_BABE, + 0x0F0F_0F0F, 0xF0F0_F0F0, 0x5555_5555, 0xAAAA_AAAA, 0x0000_00FF, 0xFF00_0000, 0x0101_0101, 0x8080_8080, + ], + [ + 0x9E37_79B9, 0x1111_1111, 0xDEAD_C0DE, 0x0BAD_F00D, 0x7FFF_FFFF, 0x0000_0000, 0xFFFF_FFFF, 0x1357_9BDF, + 0x2468_ACE0, 0xFEDC_BA98, 0x0000_0010, 0x0000_001F, 0xABCD_EF01, 0x1020_4080, 0x0F0F_F0F0, 0xC0DE_CAFE, + ], + [ + 0x1357_9BDF, 0xC0DE_CAFE, 0x0000_0000, 0xFFFF_FFFF, 0xA5A5_A5A5, 0x5A5A_5A5A, 0x0000_8000, 0x8000_0001, + 0x7777_7777, 0x8888_8888, 0x0F0F_0F0F, 0xF0F0_F0F0, 0x1234_5678, 0x8765_4321, 0xFFFF_0000, 0x0000_FFFF, + ], + ), + ]; + let u64_triples: [([u64; 8], [u64; 8], [u64; 8]); 2] = [ + ([0xF0F0_F0F0_F0F0_F0F0; 8], [0xCCCC_CCCC_CCCC_CCCC; 8], [0xAAAA_AAAA_AAAA_AAAA; 8]), + ( + [ + 0, + u64::MAX, + 1, + 1 << 63, + 0x1234_5678_9ABC_DEF0, + 0xDEAD_BEEF_CAFE_BABE, + 0x0F0F_0F0F_F0F0_F0F0, + 0x5555_AAAA_5555_AAAA, + ], + [ + 0x9E37_79B9_7F4A_7C15, + 0x1111_1111_1111_1111, + 0xDEAD_C0DE_0BAD_F00D, + u64::MAX, + 0, + 0x1357_9BDF_2468_ACE0, + 0xFEDC_BA98_7654_3210, + 0xC0DE_CAFE_C0DE_CAFE, + ], + [ + 0x1357_9BDF_1357_9BDF, + 0, + u64::MAX, + 0xA5A5_A5A5_5A5A_5A5A, + 0x8000_0000_0000_0001, + 0x7777_7777_8888_8888, + 0xFFFF_0000_0000_FFFF, + 0x0123_4567_89AB_CDEF, + ], + ), + ]; + + macro_rules! check_imm { + ($imm:expr) => {{ + const IMM: i32 = $imm; + for (a, b, c) in &u32_triples { + let got = U32x16::from_array(*a) + .ternlog::(U32x16::from_array(*b), U32x16::from_array(*c)) + .to_array(); + for i in 0..16 { + let want = reference_u64(IMM as u32, a[i] as u64, b[i] as u64, c[i] as u64) as u32; + if got[i] != want { + return Err(0x100 | IMM as u32); + } + } + } + for (a, b, c) in &u64_triples { + let got = U64x8::from_array(*a) + .ternlog::(U64x8::from_array(*b), U64x8::from_array(*c)) + .to_array(); + for i in 0..8 { + if got[i] != reference_u64(IMM as u32, a[i], b[i], c[i]) { + return Err(0x200 | IMM as u32); + } + } + } + }}; + } + // 16 × 16 = all 256 tables, each a distinct monomorphization. + macro_rules! check_row { + ($hi:expr) => { + check_imm!($hi * 16 + 0); + check_imm!($hi * 16 + 1); + check_imm!($hi * 16 + 2); + check_imm!($hi * 16 + 3); + check_imm!($hi * 16 + 4); + check_imm!($hi * 16 + 5); + check_imm!($hi * 16 + 6); + check_imm!($hi * 16 + 7); + check_imm!($hi * 16 + 8); + check_imm!($hi * 16 + 9); + check_imm!($hi * 16 + 10); + check_imm!($hi * 16 + 11); + check_imm!($hi * 16 + 12); + check_imm!($hi * 16 + 13); + check_imm!($hi * 16 + 14); + check_imm!($hi * 16 + 15); + }; + } + check_row!(0); + check_row!(1); + check_row!(2); + check_row!(3); + check_row!(4); + check_row!(5); + check_row!(6); + check_row!(7); + check_row!(8); + check_row!(9); + check_row!(10); + check_row!(11); + check_row!(12); + check_row!(13); + check_row!(14); + check_row!(15); + Ok(()) +} + +/// `U64x8` algebra on this tier's NATIVE lane type against inline scalar `u64` +/// arithmetic — the bulk mask ops (`mask_and/or/xor/andnot`) ride exactly +/// these operators. Operands populate all-zero, all-one, the sign bit and +/// mixed patterns so a lane transposition or a wrong and-not polarity fails. +/// Return codes `0x300 + op`. +fn check_u64x8_algebra() -> Result<(), u32> { + use ndarray::simd::U64x8; + let a_arr: [u64; 8] = [ + 0, + u64::MAX, + 1 << 63, + 0xF0F0_F0F0_F0F0_F0F0, + 0x1234_5678_9ABC_DEF0, + 1, + 0xDEAD_BEEF_CAFE_BABE, + 0x5555_AAAA_5555_AAAA, + ]; + let b_arr: [u64; 8] = [ + u64::MAX, + 0, + 1 << 63, + 0xCCCC_CCCC_CCCC_CCCC, + 0x0FED_CBA9_8765_4321, + 1, + 0xC0DE_CAFE_C0DE_CAFE, + 0xAAAA_5555_AAAA_5555, + ]; + let (a, b) = (U64x8::from_array(a_arr), U64x8::from_array(b_arr)); + if a.to_array() != a_arr { + return Err(0x300); + } + let mut back = [0u64; 8]; + U64x8::from_slice(&a_arr).copy_to_slice(&mut back); + if back != a_arr { + return Err(0x301); + } + let (and, or, xor, not, andnot) = + ((a & b).to_array(), (a | b).to_array(), (a ^ b).to_array(), (!a).to_array(), a.andnot(b).to_array()); + let pop = a.popcnt().to_array(); + let mut hamming = 0u64; + for i in 0..8 { + if and[i] != a_arr[i] & b_arr[i] { + return Err(0x302); + } + if or[i] != a_arr[i] | b_arr[i] { + return Err(0x303); + } + if xor[i] != a_arr[i] ^ b_arr[i] { + return Err(0x304); + } + if not[i] != !a_arr[i] { + return Err(0x305); + } + if andnot[i] != a_arr[i] & !b_arr[i] { + return Err(0x306); + } + if pop[i] != a_arr[i].count_ones() as u64 { + return Err(0x307); + } + hamming += (a_arr[i] ^ b_arr[i]).count_ones() as u64; + } + if a.xor_popcount(b) != hamming { + return Err(0x308); + } + for &n in &[0u32, 1, 7, 31, 32, 63] { + let (l, r) = (a.rotate_left(n).to_array(), a.rotate_right(n).to_array()); + for i in 0..8 { + if l[i] != a_arr[i].rotate_left(n) { + return Err(0x309); + } + if r[i] != a_arr[i].rotate_right(n) { + return Err(0x30A); + } + } + } + let mut sum = 0u64; + for &x in &a_arr { + sum = sum.wrapping_add(x); + } + if a.reduce_sum() != sum { + return Err(0x30B); + } + if (a + b).to_array() != core::array::from_fn(|i| a_arr[i].wrapping_add(b_arr[i])) { + return Err(0x30C); + } + if (a - b).to_array() != core::array::from_fn(|i| a_arr[i].wrapping_sub(b_arr[i])) { + return Err(0x30D); + } + if !(a == U64x8::from_array(a_arr)) || a == b { + return Err(0x30E); + } + Ok(()) +} + +/// `I32x16` signed compares and arithmetic on this tier's NATIVE lane type +/// against inline scalar `i32` — `gt_bitmask` is the whole ordered-compare +/// family's primitive (`gt/lt/ge/le_i32_to_mask`). Operands include +/// `i32::MIN`, `i32::MAX`, `0`, `-1` and lanes that compare EQUAL, and the +/// `u16` bit order (bit i = lane i) is asserted lane by lane. Return codes +/// `0x400 + op`. +fn check_i32x16_compare() -> Result<(), u32> { + use ndarray::simd::I32x16; + let a_arr: [i32; 16] = + [i32::MIN, i32::MAX, 0, -1, 1, -2, 7, 7, 100, -100, 0x7FFF_0000, -0x7FFF_0000, 42, -42, 2, -3]; + let b_arr: [i32; 16] = + [i32::MIN, i32::MAX, 0, 0, -1, -2, 7, 8, -100, 100, -0x7FFF_0000, 0x7FFF_0000, 41, -41, -3, 2]; + let (a, b) = (I32x16::from_array(a_arr), I32x16::from_array(b_arr)); + if a.to_array() != a_arr { + return Err(0x400); + } + let gt = a.gt_bitmask(b); + let ge0 = a.cmpge_zero_mask(); + for i in 0..16 { + if ((gt >> i) & 1 == 1) != (a_arr[i] > b_arr[i]) { + return Err(0x401); + } + if ((ge0 >> i) & 1 == 1) != (a_arr[i] >= 0) { + return Err(0x402); + } + } + if a.simd_min(b).to_array() != core::array::from_fn(|i| a_arr[i].min(b_arr[i])) { + return Err(0x403); + } + if a.simd_max(b).to_array() != core::array::from_fn(|i| a_arr[i].max(b_arr[i])) { + return Err(0x404); + } + if a.reduce_min() != *a_arr.iter().min().unwrap() || a.reduce_max() != *a_arr.iter().max().unwrap() { + return Err(0x405); + } + // `abs`/`neg` wrap at i32::MIN on every backend (release semantics). + if a.abs().to_array() != core::array::from_fn(|i| a_arr[i].wrapping_abs()) { + return Err(0x406); + } + if (-a).to_array() != core::array::from_fn(|i| a_arr[i].wrapping_neg()) { + return Err(0x407); + } + if (a + b).to_array() != core::array::from_fn(|i| a_arr[i].wrapping_add(b_arr[i])) { + return Err(0x408); + } + if (a * b).to_array() != core::array::from_fn(|i| a_arr[i].wrapping_mul(b_arr[i])) { + return Err(0x409); + } + let mut sum = 0i32; + for &x in &a_arr { + sum = sum.wrapping_add(x); + } + if a.reduce_sum() != sum { + return Err(0x40A); + } + let i16s: [i16; 16] = core::array::from_fn(|i| (i as i16 - 8) * 1000); + if I32x16::from_i16_slice(&i16s).to_array() != core::array::from_fn(|i| i16s[i] as i32) { + return Err(0x40B); + } + if a.to_i16_array() != core::array::from_fn(|i| a_arr[i] as i16) { + return Err(0x40C); + } + if !(a == I32x16::from_array(a_arr)) || a == b { + return Err(0x40D); + } + Ok(()) +} diff --git a/examples/amx_realization_report.rs b/examples/amx_realization_report.rs new file mode 100644 index 00000000..d372e4f3 --- /dev/null +++ b/examples/amx_realization_report.rs @@ -0,0 +1,28 @@ +//! Prints what the AMX path can do on THIS host — for the SIMD realization +//! matrix's `native` row, so a run where every AMX test early-returned on +//! `!amx_available()` is visibly a skip, never a pass (AMX Gotcha 9). +//! +//! AMX is runtime-gated (raw-byte / mnemonic `asm!` + `amx_available()`), so a +//! `-Ctarget-cpu=native` build always COMPILES it; whether it EXECUTED is what +//! this line records in the job log. Exits 0 either way: absence of silicon is +//! a fact about the runner, not a defect. +fn main() { + #[cfg(target_arch = "x86_64")] + { + use ndarray::simd::{amx_available, amx_report, cpu_model}; + println!("{}", amx_report()); + println!( + "amx_realization: cpu_model={:?} has_amx={} amx_available={} -> AMX tests {}", + cpu_model(), + cpu_model().has_amx(), + amx_available(), + if amx_available() { + "EXECUTED" + } else { + "SKIPPED (no AMX on this host)" + } + ); + } + #[cfg(not(target_arch = "x86_64"))] + println!("amx_realization: not x86_64 — AMX path not compiled on this target"); +} diff --git a/examples/ternlog_codegen_probe.rs b/examples/ternlog_codegen_probe.rs new file mode 100644 index 00000000..e2a88d91 --- /dev/null +++ b/examples/ternlog_codegen_probe.rs @@ -0,0 +1,125 @@ +//! Codegen witness for the mask family — the tiny OPTIMIZED surface the SIMD +//! realization matrix inspects with `--emit=asm` (`scripts/codegen-witness.sh`). +//! +//! The semantic arms of the matrix run the masking tests at `opt-level = 0`, +//! which is cheap and proves bits. It proves nothing about machine code: LLVM +//! has not run the transformations we certify. So this example is built under +//! `[profile.ci-codegen]` (opt-level 3, no debuginfo, no LTO) and its symbols +//! are checked for the instruction each backend is REQUIRED to select: +//! +//! | arm | must contain | must not contain | +//! |---|---|---| +//! | AVX-512 (`-Ctarget-cpu=x86-64-v4`) | `vpternlogq` / `vpternlogd` | — | +//! | AVX2 (`x86-64-v3`, native without avx512f) | packed `vpand`/`vpxor`/… on ymm | `vpternlog*`, GPR logic on lane data | +//! | NEON (aarch64) | `and`/`eor`/`bic`/`orr` on `v*.16b` | GPR logic on lane data | +//! +//! Every probe is `#[inline(never)]` so it survives as its own symbol, takes +//! runtime-seeded inputs through `black_box` so nothing constant-folds, and is +//! self-checked against the bit-serial ternlog definition before the assembly +//! is trusted — a packed-but-wrong body must fail here, not read as a success. +//! Calls the SHIPPED `ndarray::simd` methods, never a look-alike. + +use ndarray::simd::{mask_ternlog, ternlog, U32x16, U64x8}; +use std::hint::black_box; + +/// `U64x8::ternlog::` — the general Shannon-ladder arm. +#[inline(never)] +fn probe_ternlog_u64x8(a: U64x8, b: U64x8, c: U64x8) -> U64x8 { + a.ternlog::<{ ternlog::MAJ3 }>(b, c) +} + +/// `U32x16::ternlog::` — the immediate `simd_masking_ops` uses. +#[inline(never)] +fn probe_ternlog_u32x16(a: U32x16, b: U32x16, c: U32x16) -> U32x16 { + a.ternlog::<{ ternlog::XOR_AND }>(b, c) +} + +/// `U64x8::andnot` — the mask set-difference. +#[inline(never)] +fn probe_andnot_u64x8(a: U64x8, b: U64x8) -> U64x8 { + a.andnot(b) +} + +/// The slice-level facade op — the shape a consumer actually calls; proves +/// the ergonomic layer inlines down to the backend's realization rather than +/// adding a scalar detour. Driven at 64 words (eight full `U64x8` chunks) and +/// at 67 (plus a padded three-word tail) by the self-check. +#[inline(never)] +fn probe_mask_ternlog_slice(a: &[u64], b: &[u64], c: &[u64], dst: &mut [u64]) { + mask_ternlog::<{ ternlog::AND2_OR }>(a, b, c, dst) +} + +fn ref_ternlog(imm: i32, a: u64, b: u64, c: u64) -> u64 { + let mut out = 0u64; + for bit in 0..64 { + let idx = (((a >> bit) & 1) << 2) | (((b >> bit) & 1) << 1) | ((c >> bit) & 1); + out |= (((imm as u64) >> idx) & 1) << bit; + } + out +} + +struct SplitMix64(u64); +impl SplitMix64 { + fn next(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } +} + +fn main() { + // Runtime seed: wall clock XOR argc, so no input is a compile-time constant. + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_nanos() as u64) + .unwrap_or(0x5EED); + let mut rng = SplitMix64(black_box(nanos ^ (std::env::args().count() as u64))); + let mut acc = 0u64; + + let (a, b, c) = ( + U64x8::from_array(std::array::from_fn(|_| rng.next())), + U64x8::from_array(std::array::from_fn(|_| rng.next())), + U64x8::from_array(std::array::from_fn(|_| rng.next())), + ); + let got = probe_ternlog_u64x8(black_box(a), black_box(b), black_box(c)); + for i in 0..8 { + assert_eq!(got.to_array()[i], ref_ternlog(ternlog::MAJ3, a.to_array()[i], b.to_array()[i], c.to_array()[i])); + } + acc ^= got.reduce_sum(); + + let an = probe_andnot_u64x8(black_box(a), black_box(b)); + assert_eq!(an.to_array(), std::array::from_fn::(|i| a.to_array()[i] & !b.to_array()[i])); + acc ^= an.reduce_sum(); + + let (ua, ub, uc) = ( + U32x16::from_array(std::array::from_fn(|_| rng.next() as u32)), + U32x16::from_array(std::array::from_fn(|_| rng.next() as u32)), + U32x16::from_array(std::array::from_fn(|_| rng.next() as u32)), + ); + let ugot = probe_ternlog_u32x16(black_box(ua), black_box(ub), black_box(uc)); + for i in 0..16 { + let want = + ref_ternlog(ternlog::XOR_AND, ua.to_array()[i] as u64, ub.to_array()[i] as u64, uc.to_array()[i] as u64); + assert_eq!(ugot.to_array()[i], want as u32); + } + acc ^= ugot.reduce_sum() as u64; + + // Two lengths: 64 (eight full chunks, no tail) and 67 (eight chunks + a + // three-word padded tail), so the self-check covers the tail path the + // slice op takes on a non-multiple-of-8 mask, not only the body. + for len in [64usize, 67] { + let sa: Vec = (0..len).map(|_| rng.next()).collect(); + let sb: Vec = (0..len).map(|_| rng.next()).collect(); + let sc: Vec = (0..len).map(|_| rng.next()).collect(); + let mut sd = vec![0u64; len]; + probe_mask_ternlog_slice(black_box(&sa), black_box(&sb), black_box(&sc), black_box(&mut sd)); + for i in 0..len { + assert_eq!(sd[i], ref_ternlog(ternlog::AND2_OR, sa[i], sb[i], sc[i])); + } + acc ^= sd.iter().fold(0, |s, &x| s ^ x); + } + + println!("ternlog_codegen_probe: self-check OK, checksum = {acc:#018x}"); +} diff --git a/scripts/codegen-witness.sh b/scripts/codegen-witness.sh new file mode 100755 index 00000000..25a7b1d8 --- /dev/null +++ b/scripts/codegen-witness.sh @@ -0,0 +1,174 @@ +#!/usr/bin/env bash +# Codegen witness — does the backend the build selected actually REALIZE the +# mask family in its own instructions? Builds `examples/ternlog_codegen_probe` +# under the optimized `ci-codegen` profile with `--emit=asm`, runs the probe's +# own self-check (a packed-but-wrong body must fail before its assembly is +# trusted), then asserts per expectation: +# +# avx512 every ternlog probe symbol contains vpternlogq/vpternlogd +# avx2 no vpternlog anywhere; every ternlog probe symbol carries packed +# ymm logic and ZERO GPR logic on lane data +# neon every ternlog probe symbol carries vector logic on v*.16b and ZERO +# GPR logic +# +# Usage: scripts/codegen-witness.sh [target-triple] +# The CPU comes from the caller, never from this script — it only checks what +# the chosen CPU produced. Pass cargo's own selector through CARGO_ARGS, e.g. +# CARGO_ARGS='--config .cargo/config-v4.toml' scripts/codegen-witness.sh avx512 +# +# Why `--config` and NOT `CARGO_TARGET__RUSTFLAGS`: cargo JOINS every +# matching target..rustflags and target..rustflags entry, and the +# last `-Ctarget-cpu` wins. Measured 2026-09-14 with `cargo -v`: the env var +# form passes `-Ctarget-cpu=x86-64-v4` and THEN `.cargo/config.toml`'s +# cfg-keyed `x86-64-v3`, so the build is v3 and this witness reports 0 +# vpternlog on a "v4" build. `--config .cargo/config-v4.toml` is the same +# cfg key at higher precedence, so it is placed LAST and v4 wins. Plain +# RUSTFLAGS would win too, but it also reaches host build scripts, which is +# the SIGILL the CI job avoided by moving off it in the first place. +set -euo pipefail +EXPECT="${1:?expect: avx512|avx2|neon}" +TRIPLE="${2:-$(rustc -vV | sed -n 's/^host: //p')}" +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +TD="${CARGO_TARGET_DIR:-$ROOT/target}" +OUT="$TD/$TRIPLE/ci-codegen/examples" + +cd "$ROOT" +rm -f "$OUT"/ternlog_codegen_probe-*.s +# Deleting the old .s is only half of "never grade stale assembly": a cached +# build re-emits nothing, so the glob below would find no file at all (it did, +# on the first run after the rm landed). Touching the probe source forces the +# ONE crate that produces the .s to recompile; the library stays cached. +touch "$ROOT/examples/ternlog_codegen_probe.rs" +HOST="$(rustc -vV | sed -n 's/^host: //p')" +# A cross target emits assembly only: linking would need the foreign linker, +# and the self-check is a semantic claim the parity arms already carry for +# that platform. Natively the binary is linked AND run, so a packed-but-wrong +# body fails before its assembly is trusted. +# WITNESS_NO_RUN=1: build and inspect only, never execute — for a NATIVE triple +# on a host that cannot run the arm (an x86-64-v4 build on a runner without +# avx512f would SIGILL in the self-check). The assembly assertions still run. +if [ "$TRIPLE" = "$HOST" ] && [ "${WITNESS_NO_RUN:-0}" != "1" ]; then EMIT="asm,link"; else EMIT="asm"; fi +echo "==> building ternlog_codegen_probe (profile ci-codegen, --emit=$EMIT) for $TRIPLE, expecting $EXPECT" +# shellcheck disable=SC2086 # CARGO_ARGS is deliberately word-split +cargo ${CARGO_ARGS:-} rustc --profile ci-codegen --example ternlog_codegen_probe --target "$TRIPLE" -- --emit="$EMIT" -C debuginfo=0 +ASM="$(ls -t "$OUT"/ternlog_codegen_probe-*.s | head -1)" +[ -f "$ASM" ] || { echo "no .s produced under $OUT"; exit 2; } + +if [ "$TRIPLE" = "$HOST" ] && [ "${WITNESS_NO_RUN:-0}" != "1" ]; then + echo "==> running the probe's self-check natively" + "$TD/$TRIPLE/ci-codegen/examples/ternlog_codegen_probe" +elif [ "$TRIPLE" = "$HOST" ]; then + echo "==> WITNESS_NO_RUN=1; self-check skipped (this host cannot execute the arm — bits are NOT proven here)" +else + echo "==> cross target; self-check skipped (semantic parity is a separate arm)" +fi + +# Per-symbol attribution: function labels only (never LLVM's .L* basic-block +# labels — counting those fragments a body into 4-op stubs and misreports it +# as scalarised; measured on scripts/neon-asm-rung3.sh 2026-09-14). +# +# Two rules, because the probes have two shapes: +# * register probes (probe_ternlog_u64x8 / _u32x16 / probe_andnot_u64x8) — +# straight-line, no loop: packed logic present AND ZERO GPR logic. +# * the slice probe (probe_mask_ternlog_slice) — a loop over 64 words: +# packed logic present AND GPR logic bounded by SLICE_GPR_CAP. The count +# rule cannot tell lane data from index arithmetic — it counts every +# and/or/xor/andn/not with a GPR operand (`xorl %r, %r` register clears +# excluded). What remains after `simd_masking_ops` moved its tails onto +# `pad_tail` (zero-pad into one register, same packed op as the body) was +# read by hand, 2026-09-14: `andl $7` (tail length) and `andq $-64`-style +# chunk-byte rounding — 4 on v3, 2 on aarch64. History of the same probe: +# master's scalar index-loop tail measured 9 (v3) / 6 (aarch64); the first +# `as_chunks` cut with an exact-length scalar tail measured 4 / 16 — LLVM +# fully unrolled the aarch64 tail into 7 × (and, orr) on GPRs, which is +# what the padding removed. The cap sits at the measured floor plus two: +# a 1-word scalar peel (2 ops) re-appearing on v3 would reach exactly the +# cap; anything larger fails. So a peel IS a regression here and the gate +# says so. The "packed present" half is the discriminating one on x86 +# (a scalarised body has no ymm/zmm logic at all); on aarch64 the vec +# rule additionally requires a vector-register operand for the same +# reason (`and x8, x9, #7` and `and v0.16b, ...` share a mnemonic). +SLICE_GPR_CAP=6 +# The scalar rule inspects EVERY operand field ($2..$NF), not just the first: +# `andq $-8, %r10` has its register in $3 and `andq (%rdi,%rax,8), %r9` its +# register in $3 too, so a `$2`-only rule undercounted exactly the two forms +# the slice cap exists to bound (CodeRabbit, PR #306). Every required probe +# is always printed (the loops below fail on a missing row rather than +# skipping it), so a symbol the compiler folded away cannot pass by absence. +REQUIRED_PROBES="probe_ternlog_u64x8 probe_ternlog_u32x16 probe_andnot_u64x8 probe_mask_ternlog_slice" +report() { + awk -v vec="$1" -v sca="$2" -v vecop="${3:-}" -v req="$REQUIRED_PROBES" ' + BEGIN { n = split(req, r, " "); for (i = 1; i <= n; i++) want[r[i]] = 1 } + /^[A-Za-z_$][A-Za-z0-9_.$]*:/ { sym=$1; for (k in want) if (index(sym, k) && !(k in seen)) seen[k] = sym } + sym ~ /probe_/ && $1 ~ vec { + # On aarch64 `and`/`orr`/`eor`/`bic` name BOTH the vector and the GPR + # form; only an operand of the shape `v3.16b` makes it vector logic. + # x86 packed mnemonics (`vpand`, `vandps`, …) have no GPR form, so + # the operand test is a no-op there. + if (vecop == "") { v[sym]++ } + else { for (f = 2; f <= NF; f++) if ($f ~ vecop) { v[sym]++; break } } + } + sym ~ /probe_/ && $1 ~ sca { + # `xorl %eax, %eax` is the register-clear idiom, not logic on data: + # identical source and destination means the value is discarded. + a = $2; sub(/,$/, "", a) + if ($1 ~ /^xor/ && NF == 3 && a == $3) next + hit = 0 + for (f = 2; f <= NF; f++) if ($f ~ /%[re][a-z0-9]+|^[wx][0-9]+,?$/) hit = 1 + if (hit) s[sym]++ + } + END { + for (k in want) if (!(k in seen)) printf "%6d vec %6d sca MISSING:%s\n", 0, 0, k + for (k in seen) { sym = seen[k]; printf "%6d vec %6d sca %s\n", v[sym]+0, s[sym]+0, sym } + } + ' "$ASM" | sort -k5 +} + +fail=0 +case "$EXPECT" in + avx512) + echo "==> ternlog symbols must select vpternlog{q,d}" + for sym in probe_ternlog_u64x8 probe_ternlog_u32x16 probe_mask_ternlog_slice; do + n=$(awk -v s="$sym" '/^[A-Za-z_$][A-Za-z0-9_.$]*:/ { sym=$1 } sym ~ s && $1 ~ /^vpternlog[qd]$/ { c++ } END { print c+0 }' "$ASM") + echo " $sym: $n vpternlog" + [ "$n" -ge 1 ] || { echo " FAIL: $sym has no vpternlog on an AVX-512 build"; fail=1; } + done + ;; + avx2) + echo "==> AVX2 build must not contain vpternlog anywhere" + n=$(grep -cE '^\s+vpternlog' "$ASM" || true) + [ "$n" -eq 0 ] || { echo " FAIL: $n vpternlog instructions on an AVX2 build (wrong arm selected)"; fail=1; } + echo "==> per-symbol packed (ymm logic) vs GPR logic on lane data" + report '^(vpand|vpandn|vpor|vpxor|vandps|vandnps|vorps|vxorps|vandpd|vandnpd|vorpd|vxorpd)$' '^(and[lq]?|or[lq]?|xor[lq]?|andn[lq]?|not[lq]?)$' + while read -r v _ s _ sym; do + case "$sym" in + MISSING:*) + echo " FAIL: required probe $sym has no symbol in the assembly"; fail=1 ;; + *probe_mask_ternlog_slice*) + [ "$v" -ge 2 ] || { echo " FAIL: $sym has no packed logic (facade layer scalarised)"; fail=1; } + [ "$s" -le "$SLICE_GPR_CAP" ] || { echo " FAIL: $sym carries $s GPR logic ops (cap $SLICE_GPR_CAP = measured index arithmetic + 2; a scalar tail peel is a regression)"; fail=1; } ;; + *probe_ternlog*|*probe_andnot*) + [ "$v" -ge 1 ] || { echo " FAIL: $sym has no packed logic"; fail=1; } + [ "$s" -eq 0 ] || { echo " FAIL: $sym carries $s GPR logic ops on lane data"; fail=1; } ;; + esac + done < <(report '^(vpand|vpandn|vpor|vpxor|vandps|vandnps|vorps|vxorps|vandpd|vandnpd|vorpd|vxorpd)$' '^(and[lq]?|or[lq]?|xor[lq]?|andn[lq]?|not[lq]?)$') + ;; + neon) + echo "==> per-symbol vector (v*.16b) vs GPR logic" + report '^(and|orr|eor|bic|orn|eon|mvn|not|bif|bit|bsl)$' '^(and|orr|eor|bic|orn|eon|mvn)$' '^v[0-9]+\.(16b|8b|4s|2d|8h)' + while read -r v _ s _ sym; do + case "$sym" in + MISSING:*) + echo " FAIL: required probe $sym has no symbol in the assembly"; fail=1 ;; + *probe_mask_ternlog_slice*) + [ "$v" -ge 2 ] || { echo " FAIL: $sym has no vector logic (facade layer scalarised)"; fail=1; } + [ "$s" -le "$SLICE_GPR_CAP" ] || { echo " FAIL: $sym carries $s GPR logic ops (cap $SLICE_GPR_CAP = measured index arithmetic + 2; a scalar tail peel is a regression)"; fail=1; } ;; + *probe_ternlog*|*probe_andnot*) + [ "$v" -ge 1 ] || { echo " FAIL: $sym has no vector logic"; fail=1; } + [ "$s" -eq 0 ] || { echo " FAIL: $sym carries $s GPR logic ops"; fail=1; } ;; + esac + done < <(report '^(and|orr|eor|bic|orn|eon|mvn|not|bif|bit|bsl)$' '^(and|orr|eor|bic|orn|eon|mvn)$' '^v[0-9]+\.(16b|8b|4s|2d|8h)') + ;; + *) echo "unknown expectation: $EXPECT"; exit 2 ;; +esac +[ "$fail" -eq 0 ] && echo "codegen witness ($EXPECT): PASS" || { echo "codegen witness ($EXPECT): FAIL"; exit 1; } diff --git a/scripts/masking-parity.sh b/scripts/masking-parity.sh new file mode 100755 index 00000000..4dff97cb --- /dev/null +++ b/scripts/masking-parity.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# Masking parity — build `crates/simd-masking-parity` (the ONE facade-only +# parity program) under a realization selector and run it. The program has no +# idea which backend `simd.rs` picked; this script only chooses the build. +# +# scripts/masking-parity.sh native host triple, whatever .cargo/config* selects +# scripts/masking-parity.sh nightly `cargo +nightly --features nightly-simd` +# scripts/masking-parity.sh wasm wasm32 +simd128 under node (simd_wasm arm) +# scripts/masking-parity.sh wasm-scalar wasm32 WITHOUT simd128 under node = the scalar arm +# scripts/masking-parity.sh neon-qemu aarch64 cross-build run under qemu-aarch64-static +# +# Extra cargo arguments (e.g. `--config .cargo/config-v4.toml` for the v4 +# realization on the native row) pass through CARGO_ARGS, see +# scripts/codegen-witness.sh for why that form and not the env var. +set -euo pipefail +ARM="${1:?native|nightly|wasm|wasm-scalar|neon-qemu}" +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +MANIFEST="$ROOT/crates/simd-masking-parity/Cargo.toml" +TD="${CARGO_TARGET_DIR:-$ROOT/crates/simd-masking-parity/target}" +export CARGO_TARGET_DIR="$TD" +cd "$ROOT" + +# shellcheck disable=SC2086 +case "$ARM" in + native) + # `env -u RUSTFLAGS`: a workflow-global RUSTFLAGS (CI sets "-D warnings") + # REPLACES every cargo-config rustflags entry, so a `--config + # .cargo/config-v4.toml` passed through CARGO_ARGS would silently lose its + # `-Ctarget-cpu=x86-64-v4` and this arm would measure v3 while claiming + # v4 — the exact trap the tier4 CI job hit. Clearing it lets the config win. + env -u RUSTFLAGS cargo ${CARGO_ARGS:-} build --release --manifest-path "$MANIFEST" --bin simd-masking-parity + "$TD/release/simd-masking-parity" + ;; + nightly) + cargo +nightly ${CARGO_ARGS:-} build --release --manifest-path "$MANIFEST" --bin simd-masking-parity --features nightly-simd + "$TD/release/simd-masking-parity" + ;; + wasm|wasm-scalar) + FLAGS=""; [ "$ARM" = wasm ] && FLAGS="-C target-feature=+simd128" + RUSTFLAGS="$FLAGS" cargo ${CARGO_ARGS:-} build --release --lib --manifest-path "$MANIFEST" --target wasm32-unknown-unknown + echo "==> $ARM (RUSTFLAGS='$FLAGS') under node" + node "$ROOT/crates/simd-masking-parity/run.mjs" "$TD/wasm32-unknown-unknown/release/simd_masking_parity.wasm" + ;; + neon-qemu) + QEMU="${QEMU_AARCH64:-qemu-aarch64-static}" + SYSROOT="${AARCH64_SYSROOT:-/usr/aarch64-linux-gnu}" + CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER="${CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER:-aarch64-linux-gnu-gcc}" \ + cargo ${CARGO_ARGS:-} build --release --manifest-path "$MANIFEST" --bin simd-masking-parity --target aarch64-unknown-linux-gnu + "$QEMU" -L "$SYSROOT" "$TD/aarch64-unknown-linux-gnu/release/simd-masking-parity" + ;; + *) echo "unknown arm: $ARM"; exit 2 ;; +esac +echo "masking parity ($ARM): PASS" diff --git a/scripts/neon-asm-rung3.sh b/scripts/neon-asm-rung3.sh new file mode 100755 index 00000000..bd10f9d7 --- /dev/null +++ b/scripts/neon-asm-rung3.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# AArch64 acceptance ladder, rung 3: does the cross-compiled harness assembly +# SELECT NEON vector logical instructions for the mask lane, or scalarise? +# +# Rung 1 (compile) and rung 2 (qemu parity) can both pass on a body that +# LLVM scalarised — measured 2026-09-13 on the first generated NEON ternlog +# (a per-lane loop through to_array/from_array): 536 scalar vs 4 vector ops. +# This script is that inspection, made re-runnable and SYMMETRIC: the same +# mnemonic set is counted on vector registers (v*.16b / v*.8b) and on +# general-purpose registers (w*/x*), and every count is attributed to the +# symbol it appears in so "scaffolding" is a measurement, not a claim. +# Attribution is per FUNCTION symbol: LLVM's local `.LBB*` basic-block labels +# are deliberately not symbol boundaries — treating them as such fragmented a +# 684-vector-op ternlog body into hundreds of 4-op fragments and misreported +# the function as scalarised (measured 2026-09-14, first run of this script). +# +# Requires: `rustup target add aarch64-unknown-linux-gnu`. No linker or qemu +# needed — `--emit=asm` stops before the link step. +set -euo pipefail +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +MANIFEST="$ROOT/crates/neon-simd-parity/Cargo.toml" +TARGET="aarch64-unknown-linux-gnu" +TD="${CARGO_TARGET_DIR:-$ROOT/target-aarch64}" + +echo "==> emitting aarch64 assembly of neon-simd-parity (link step skipped)" +# Stale-assembly guard: remove any previous .s FIRST and let a build failure +# fail the script — with `|| true` and suppressed output, a broken build would +# leave an older .s for `ls -t` to pick up and the gate would PASS on assembly +# that does not match the source (CodeRabbit, PR #306). +rm -f "$TD/$TARGET/release/deps/"neon_simd_parity-*.s +# A cached build re-emits no .s after the rm, so force the harness crate to +# recompile (the library stays cached); see scripts/codegen-witness.sh. +touch "$ROOT/crates/neon-simd-parity/src/main.rs" +CARGO_TARGET_DIR="$TD" cargo rustc --release --manifest-path "$MANIFEST" --target "$TARGET" \ + -- --emit=asm -C debuginfo=0 +ASM="$(ls -t "$TD/$TARGET/release/deps/"neon_simd_parity-*.s | head -1)" +[ -f "$ASM" ] || { echo "no .s produced"; exit 2; } +echo "asm: $ASM" + +LOGIC='(and|orr|eor|bic|orn|eon|mvn|not|bif|bit|bsl)' +VEC=$(grep -cE "^\s+${LOGIC}\s+v[0-9]+\.(16b|8b)" "$ASM" || true) +SCA=$(grep -cE "^\s+${LOGIC}\s+[wx][0-9]+," "$ASM" || true) +echo "vector logical ops (v-regs): $VEC" +echo "scalar logical ops (w/x-regs): $SCA" + +echo "==> per-symbol attribution (top 12 by vector count, then any symbol with scalar ops)" +awk -v logic="$LOGIC" ' + /^[A-Za-z_$][A-Za-z0-9_.$]*:/ { sym=$1 } # NOT .L*: LLVM local (basic-block) labels stay inside their function + $1 ~ "^"logic"$" && $2 ~ /^v[0-9]+\.(16b|8b)/ { v[sym]++ } + $1 ~ "^"logic"$" && $2 ~ /^[wx][0-9]+,/ { s[sym]++ } + END { for (k in v) printf "%6d vec %6d sca %s\n", v[k], s[k]+0, k; for (k in s) if (!(k in v)) printf "%6d vec %6d sca %s\n", 0, s[k], k } +' "$ASM" | sort -rn | head -12 + +# Gate: the ternlog / mask kernels must be vector-dominant. Symbols whose +# demangled name contains "ternlog" must carry more vector than scalar logic — +# EXCEPT the scalar reference oracles (`*reference*`), which are scalar by +# design: the parity check is SIMD-vs-scalar, so a vectorised oracle would +# compare the backend against itself. +echo "==> gate: ternlog symbols vector-dominant" +BAD=$(awk -v logic="$LOGIC" ' + /^[A-Za-z_$][A-Za-z0-9_.$]*:/ { sym=$1 } # NOT .L*: LLVM local (basic-block) labels stay inside their function + $1 ~ "^"logic"$" && $2 ~ /^v[0-9]+\.(16b|8b)/ { v[sym]++ } + $1 ~ "^"logic"$" && $2 ~ /^[wx][0-9]+,/ { s[sym]++ } + END { for (k in s) if (k ~ /ternlog/ && k !~ /reference/ && s[k] > v[k]+0) print k } +' "$ASM") +if [ -n "$BAD" ]; then echo "SCALARISED ternlog symbols:"; echo "$BAD"; exit 1; fi +echo "rung 3: PASS ($VEC vector / $SCA scalar logical ops)" diff --git a/src/aabb.rs b/src/aabb.rs index 180dbf1a..3c90dcaa 100644 --- a/src/aabb.rs +++ b/src/aabb.rs @@ -226,7 +226,8 @@ unsafe fn aabb_intersect_batch_avx512(query: &Aabb, candidates: &[Aabb]) -> Vec< let m5 = q_min_z.simd_le(v_c_max_z); let m6 = q_max_z.simd_ge(v_c_min_z); - let all = m1.0 & m2.0 & m3.0 & m4.0 & m5.0 & m6.0; + let all = + m1.to_bitmask() & m2.to_bitmask() & m3.to_bitmask() & m4.to_bitmask() & m5.to_bitmask() & m6.to_bitmask(); for i in 0..16 { result.push((all >> i) & 1 != 0); @@ -401,7 +402,7 @@ unsafe fn ray_aabb_slab_test_avx512(ray: &Ray, aabbs: &[Aabb]) -> (Vec, Ve // hit = t_enter <= t_exit AND t_exit >= 0 let m_le = t_enter.simd_le(t_exit); let m_ge = t_exit.simd_ge(zero); - let hit_mask = m_le.0 & m_ge.0; + let hit_mask = m_le.to_bitmask() & m_ge.to_bitmask(); // Clamp t_enter to 0 for origins inside box let t_enter_clamped = t_enter.simd_max(zero); diff --git a/src/hpc/amx_ops.rs b/src/hpc/amx_ops.rs new file mode 100644 index 00000000..390fa903 --- /dev/null +++ b/src/hpc/amx_ops.rs @@ -0,0 +1,1147 @@ +//! The full AMX instruction surface as mnemonics — every tile op LLVM's +//! `X86InstrAMX.td` defines, on stable Rust, with the tile numbers as +//! `const` operands. +//! +//! [`amx_matmul`](super::amx_matmul) carries the GEMM subset (`TILEZERO` +//! tmm0..3, `TILELOADD`, `TILESTORED`, `TDPBUSD`, `TDPBF16PS`) as hand-written +//! `.byte` sequences, because in 1.94 that was the only way. Measured on +//! 1.98.1 (LLVM 22.1.8): the integrated assembler accepts every AMX mnemonic +//! inside `asm!` WITHOUT any target feature, and `asm_const` lets the tile +//! index be a generic parameter, so `tilezero tmm{t}` assembles for all eight +//! tiles from one body. This module is that surface. The `.byte` tables stay +//! where they are; the encoding tests below read this module's emitted bytes +//! out of the test binary and pin the GEMM subset — TILEZERO tmm0, TILERELEASE, +//! TDPBUSD and TDPBF16PS in the kernel's `(0, 2, 1)` placement — to the +//! EMR-validated table, so THOSE four cannot disagree silently. Every other +//! pinned op is pinned to LLVM 22.1.8's own emission (a drift guard); the +//! `amx_matmul` load/store rows are not cross-pinned. +//! +//! # Operand order — the "mirror" resolved +//! +//! Intel syntax and LLVM's `MRMSrcReg4VOp3` agree: `tdpbusd tmmD, tmmS1, tmmS2` +//! encodes `D` in ModRM.reg, `S1` in ModRM.rm and `S2` in VEX.vvvv, and the +//! semantics are `D[m][n] += S1[m][k] · S2[k][n]` with S1 the plain M×K +//! operand and S2 the VNNI-packed K×N operand; the letters in `TDPBD` +//! name S1's and S2's signedness in that order. The repo's validated table +//! entry `C4 E2 71 5E C2` (rm = tmm2, vvvv = tmm1) is therefore the mnemonic +//! `tdpbusd tmm0, tmm2, tmm1`, i.e. [`tdpbusd::<0, 2, 1>`] — exactly the +//! kernel's placement (plain u8 A in tmm2, VNNI i8 B in tmm1, per +//! `AMX_GOTCHAS.md` Gotcha 12). Nothing was mirrored; the byte table had +//! been read as `(dst, vvvv, rm)`. With mnemonics the question does not +//! arise: name the tiles in Intel order and the assembler does the rest. +//! +//! # Feature tiers (CPUID bits per LLVM `Host.cpp`) +//! +//! | tier | ops | CPUID | first silicon | +//! |---|---|---|---| +//! | AMX-TILE | config, zero, load/store, release | 7.0:EDX[24] | Sapphire Rapids | +//! | AMX-INT8 | `tdpb{ss,su,us,uu}d` | 7.0:EDX[25] | Sapphire Rapids | +//! | AMX-BF16 | `tdpbf16ps` | 7.0:EDX[22] | Sapphire Rapids | +//! | AMX-FP16 | `tdpfp16ps` | 7.1:EAX[21] | Granite Rapids | +//! | AMX-COMPLEX | `tcmm{im,rl}fp16ps` | 7.1:EDX[8] | Granite Rapids-D | +//! | AMX-FP8 | `tdp{b,bh,hb,h}f8ps` | 1E.1:EAX[4] | Diamond Rapids | +//! | AMX-TF32 | `tmmultf32ps` | 1E.1:EAX[6] | Diamond Rapids (see note) | +//! | AMX-AVX512 | `tcvtrow*`, `tilemovrow` | 1E.1:EAX[7] | Diamond Rapids | +//! | AMX-MOVRS | `tileloaddrs{,t1}` | 1E.1:EAX[8] | Diamond Rapids | +//! +//! Note on TF32: LLVM `main` has removed `amx-tf32` (and `amx-transpose`) +//! from both the assembler and `Host.cpp`. The 22.1.8 assembler in the +//! stable toolchain still accepts `tmmultf32ps`, but nightly's LLVM 23 +//! already rejects the mnemonic (measured 2026-09-14: `invalid instruction +//! mnemonic 'tmmultf32ps'` on `1.100.0-nightly` / LLVM 23.1.1 — the lib +//! builds because the wrapper is generic, but the first instantiation +//! fails). So that ONE wrapper is emitted as its fixed ISA byte encoding +//! (`VEX.128.66.0F38.W0 48 /r`) instead of the mnemonic — the encoding is +//! defined by the ISA, not by which LLVM still knows the name — and it is +//! gated on the CPUID bit the older `Host.cpp` used. Treat it as CLAIMED, +//! never executed. +//! +//! # What has executed +//! +//! Only the AMX-TILE / INT8 / BF16 tier has ever run in this workspace +//! (Emerald Rapids, `AMX_GOTCHAS.md`). Every other tier here is +//! assembler-verified — the bytes are what LLVM emits for the mnemonic — and +//! NOT execution-verified: no Granite/Diamond Rapids host has run them. The +//! detection API says which tier a host has; the caller must still gate on it. +//! +//! # Safety model +//! +//! Every op is `unsafe` with three preconditions, and the FIRST is split by +//! tier — `amx_available()` is the INT8 gate (AMX-TILE + AMX-INT8 + OS + +//! permission) and must NOT be the precondition for every op, because a host +//! or hypervisor can expose TILE with BF16 / FP16 / FP8 while masking INT8: +//! +//! 1. **Tile state + permission**: [`crate::simd_amx::amx_tile_available`] +//! returned `true` (AMX-TILE, XSAVE, tile XSTATE, XTILEDATA permission — +//! no compute-tier bit). Sufficient on its own for the tile-STATE ops: +//! `ldtilecfg`, `sttilecfg`, `tilezero`, `tileloadd`, `tileloaddt1`, +//! `tilestored`, `tilerelease`. NOT for `tileloaddrs`/`tileloaddrst1`, +//! which are AMX-MOVRS and take gate 2 like any other tier. +//! 2. **Tier**: the compute op's tier is advertised — INT8 ops via +//! [`super::amx_matmul::amx_available`] (which is gate 1 plus the INT8 +//! bit), every other op via gate 1 AND its [`AmxFeatures`] bit +//! (`bf16`, `fp16`, `complex`, `fp8`, `tf32`, `avx512`, `movrs`). +//! 3. `LDTILECFG` has been executed with a config covering every tile named, +//! and pointers/strides are valid for the configured rows × colsb. +//! +//! Tile-operand aliasing (Gotcha 11, `#UD` → SIGILL) is a COMPILE error here: +//! every three-tile op asserts `D != S1 != S2` in a `const` block. + +use core::arch::asm; + +// ── AMX-TILE: configuration and data movement ─────────────────────────────── + +/// `LDTILECFG [cfg]` — load the 64-byte tile configuration. +/// +/// # Safety +/// `cfg` must point to 64 readable bytes, 64-byte aligned (`TileConfig`), with +/// a valid palette and in-range rows/colsb (Gotchas 2, 6, 7). +/// +/// # Examples +/// +/// ```rust,no_run +/// use ndarray::hpc::amx_matmul::TileConfig; +/// use ndarray::hpc::amx_ops::{ldtilecfg, tilerelease}; +/// use ndarray::simd_amx::amx_tile_available; +/// +/// if amx_tile_available() { +/// let cfg = TileConfig::for_dpbusd(64); +/// // SAFETY: tile permission held (checked above); `cfg` is 64 aligned bytes +/// // with palette 1 and in-range shapes, covering tiles 0-2. +/// unsafe { +/// ldtilecfg(cfg.data.as_ptr()); +/// tilerelease(); +/// } +/// } +/// ``` +#[inline(always)] +pub unsafe fn ldtilecfg(cfg: *const u8) { + asm!("ldtilecfg [{c}]", c = in(reg) cfg, options(nostack, readonly)); +} + +/// `STTILECFG [cfg]` — store the current tile configuration (64 bytes). +/// +/// # Safety +/// `cfg` must point to 64 writable, 64-byte-aligned bytes. +/// +/// # Examples +/// +/// ```rust,no_run +/// use ndarray::hpc::amx_matmul::TileConfig; +/// use ndarray::hpc::amx_ops::{ldtilecfg, tilerelease, sttilecfg}; +/// use ndarray::simd_amx::amx_tile_available; +/// +/// if amx_tile_available() { +/// let cfg = TileConfig::for_dpbusd(64); +/// // SAFETY: tile permission held (checked above); `cfg` is 64 aligned bytes +/// // with palette 1 and in-range shapes, covering tiles 0-2. +/// unsafe { +/// ldtilecfg(cfg.data.as_ptr()); +/// let mut back = TileConfig { data: [0u8; 64] }; +/// sttilecfg(back.data.as_mut_ptr()); +/// assert_eq!(back.data[0], 1, "palette 1 reads back"); +/// tilerelease(); +/// } +/// } +/// ``` +#[inline(always)] +pub unsafe fn sttilecfg(cfg: *mut u8) { + asm!("sttilecfg [{c}]", c = in(reg) cfg, options(nostack)); +} + +/// `TILERELEASE` — return all tiles to the init state. +/// +/// # Safety +/// AMX must be available; no tile may be needed afterwards. +/// +/// # Examples +/// +/// ```rust,no_run +/// use ndarray::hpc::amx_matmul::TileConfig; +/// use ndarray::hpc::amx_ops::{ldtilecfg, tilerelease}; +/// use ndarray::simd_amx::amx_tile_available; +/// +/// if amx_tile_available() { +/// let cfg = TileConfig::for_dpbusd(64); +/// // SAFETY: tile permission held (checked above); `cfg` is 64 aligned bytes +/// // with palette 1 and in-range shapes, covering tiles 0-2. +/// unsafe { +/// ldtilecfg(cfg.data.as_ptr()); +/// tilerelease(); +/// } +/// } +/// ``` +#[inline(always)] +pub unsafe fn tilerelease() { + asm!("tilerelease", options(nostack, nomem)); +} + +/// `TILEZERO tmm{T}` for any of the eight tiles. +/// +/// # Safety +/// Tiles configured; `T < 8`. +/// +/// # Examples +/// +/// ```rust,no_run +/// use ndarray::hpc::amx_matmul::TileConfig; +/// use ndarray::hpc::amx_ops::{ldtilecfg, tilerelease, tilezero}; +/// use ndarray::simd_amx::amx_tile_available; +/// +/// if amx_tile_available() { +/// let cfg = TileConfig::for_dpbusd(64); +/// // SAFETY: tile permission held (checked above); `cfg` is 64 aligned bytes +/// // with palette 1 and in-range shapes, covering tiles 0-2. +/// unsafe { +/// ldtilecfg(cfg.data.as_ptr()); +/// tilezero::<0>(); +/// tilerelease(); +/// } +/// } +/// ``` +#[inline(always)] +pub unsafe fn tilezero() { + const { assert!(T < 8) } + asm!("tilezero tmm{t}", t = const T, options(nostack, nomem)); +} + +/// `TILELOADD tmm{T}, [base + stride]` — load a tile, one row per `stride` +/// bytes. +/// +/// # Safety +/// `base` must be readable for `rows × colsb` of tile `T` at the given row +/// stride; tile configured. +/// +/// # Examples +/// +/// ```rust,no_run +/// use ndarray::hpc::amx_matmul::TileConfig; +/// use ndarray::hpc::amx_ops::{ldtilecfg, tilerelease, tileloadd}; +/// use ndarray::simd_amx::amx_tile_available; +/// +/// if amx_tile_available() { +/// let cfg = TileConfig::for_dpbusd(64); +/// // SAFETY: tile permission held (checked above); `cfg` is 64 aligned bytes +/// // with palette 1 and in-range shapes, covering tiles 0-2. +/// unsafe { +/// ldtilecfg(cfg.data.as_ptr()); +/// // tmm2 is the 16-row × 64-byte M×K operand: one 64-byte row per stride. +/// let a = [0u8; 16 * 64]; +/// tileloadd::<2>(a.as_ptr(), 64); +/// tilerelease(); +/// } +/// } +/// ``` +#[inline(always)] +pub unsafe fn tileloadd(base: *const u8, stride: usize) { + const { assert!(T < 8) } + asm!("tileloadd tmm{t}, [{b} + {s}*1]", t = const T, b = in(reg) base, s = in(reg) stride, options(nostack, readonly)); +} + +/// `TILELOADDT1` — same as [`tileloadd`] with the non-temporal (T1) hint. +/// +/// # Safety +/// As [`tileloadd`]. +/// +/// # Examples +/// +/// ```rust,no_run +/// use ndarray::hpc::amx_matmul::TileConfig; +/// use ndarray::hpc::amx_ops::{ldtilecfg, tilerelease, tileloaddt1}; +/// use ndarray::simd_amx::amx_tile_available; +/// +/// if amx_tile_available() { +/// let cfg = TileConfig::for_dpbusd(64); +/// // SAFETY: tile permission held (checked above); `cfg` is 64 aligned bytes +/// // with palette 1 and in-range shapes, covering tiles 0-2. +/// unsafe { +/// ldtilecfg(cfg.data.as_ptr()); +/// let a = [0u8; 16 * 64]; +/// tileloaddt1::<2>(a.as_ptr(), 64); +/// tilerelease(); +/// } +/// } +/// ``` +#[inline(always)] +pub unsafe fn tileloaddt1(base: *const u8, stride: usize) { + const { assert!(T < 8) } + asm!("tileloaddt1 tmm{t}, [{b} + {s}*1]", t = const T, b = in(reg) base, s = in(reg) stride, options(nostack, readonly)); +} + +/// `TILESTORED [base + stride], tmm{T}` — store a tile. +/// +/// # Safety +/// `base` must be writable for `rows × colsb` of tile `T` at the given row +/// stride; tile configured. +/// +/// # Examples +/// +/// ```rust,no_run +/// use ndarray::hpc::amx_matmul::TileConfig; +/// use ndarray::hpc::amx_ops::{ldtilecfg, tilerelease, tilestored, tilezero}; +/// use ndarray::simd_amx::amx_tile_available; +/// +/// if amx_tile_available() { +/// let cfg = TileConfig::for_dpbusd(64); +/// // SAFETY: tile permission held (checked above); `cfg` is 64 aligned bytes +/// // with palette 1 and in-range shapes, covering tiles 0-2. +/// unsafe { +/// ldtilecfg(cfg.data.as_ptr()); +/// // tmm0 is the 16×16 i32 accumulator: 16 rows of 64 bytes. +/// let mut c = [0i32; 16 * 16]; +/// tilezero::<0>(); +/// tilestored::<0>(c.as_mut_ptr().cast::(), 64); +/// assert!(c.iter().all(|&x| x == 0)); +/// tilerelease(); +/// } +/// } +/// ``` +#[inline(always)] +pub unsafe fn tilestored(base: *mut u8, stride: usize) { + const { assert!(T < 8) } + asm!("tilestored [{b} + {s}*1], tmm{t}", t = const T, b = in(reg) base, s = in(reg) stride, options(nostack)); +} + +// ── AMX-MOVRS (Diamond Rapids): read-shared loads ─────────────────────────── + +/// `TILELOADDRS tmm{T}, [base + stride]` — load with the read-shared hint +/// (AMX-MOVRS). Assembler-verified, not execution-verified. +/// +/// # Safety +/// As [`tileloadd`], and the host must report [`AmxFeatures::movrs`]. +/// +/// # Examples +/// +/// ```rust,no_run +/// use ndarray::hpc::amx_matmul::TileConfig; +/// use ndarray::hpc::amx_ops::{amx_features, ldtilecfg, tilerelease, tileloaddrs}; +/// use ndarray::simd_amx::amx_tile_available; +/// +/// // Tile state AND the MOVRS tier — the INT8 gate says nothing about MOVRS. +/// if amx_tile_available() && amx_features().movrs { +/// let cfg = TileConfig::for_dpbusd(64); +/// let a = [0u8; 16 * 64]; +/// // SAFETY: tile permission held, config covers tile 2, `a` is 16 rows × 64 B. +/// unsafe { +/// ldtilecfg(cfg.data.as_ptr()); +/// tileloaddrs::<2>(a.as_ptr(), 64); +/// tilerelease(); +/// } +/// } +/// ``` +#[inline(always)] +pub unsafe fn tileloaddrs(base: *const u8, stride: usize) { + const { assert!(T < 8) } + asm!("tileloaddrs tmm{t}, [{b} + {s}*1]", t = const T, b = in(reg) base, s = in(reg) stride, options(nostack, readonly)); +} + +/// `TILELOADDRST1` — read-shared load with the T1 hint (AMX-MOVRS). +/// +/// # Safety +/// As [`tileloaddrs`]. +/// +/// # Examples +/// +/// ```rust,no_run +/// use ndarray::hpc::amx_matmul::TileConfig; +/// use ndarray::hpc::amx_ops::{amx_features, ldtilecfg, tilerelease, tileloaddrst1}; +/// use ndarray::simd_amx::amx_tile_available; +/// +/// // Tile state AND the MOVRS tier — the INT8 gate says nothing about MOVRS. +/// if amx_tile_available() && amx_features().movrs { +/// let cfg = TileConfig::for_dpbusd(64); +/// let a = [0u8; 16 * 64]; +/// // SAFETY: tile permission held, config covers tile 2, `a` is 16 rows × 64 B. +/// unsafe { +/// ldtilecfg(cfg.data.as_ptr()); +/// tileloaddrst1::<2>(a.as_ptr(), 64); +/// tilerelease(); +/// } +/// } +/// ``` +#[inline(always)] +pub unsafe fn tileloaddrst1(base: *const u8, stride: usize) { + const { assert!(T < 8) } + asm!("tileloaddrst1 tmm{t}, [{b} + {s}*1]", t = const T, b = in(reg) base, s = in(reg) stride, options(nostack, readonly)); +} + +// ── Three-tile dot products: D += S1 · S2 ─────────────────────────────────── + +macro_rules! tdp3 { + ($(#[$m:meta])* $name:ident, $mn:literal, $tier:ident) => { + $(#[$m])* + /// + /// `D += S1 · S2`; S1 is the plain M×K operand (ModRM.rm), S2 the + /// VNNI-packed K×N operand (VEX.vvvv). The three tiles must be + /// distinct — enforced at compile time. + /// + /// # Safety + /// Tiles configured with compatible shapes, + /// [`crate::simd_amx::amx_tile_available`] true, and the host must + #[doc = concat!("report this op's tier: [`AmxFeatures::", stringify!($tier), "`].")] + /// + /// # Examples + /// + /// Gate on the tile state AND this op's own tier — never on the INT8 + /// gate for a non-INT8 op — then run it on the three distinct tiles + /// the GEMM config lays out (`C → tmm0`, VNNI K×N → tmm1, M×K → tmm2). + /// + /// ```rust,no_run + /// use ndarray::hpc::amx_matmul::TileConfig; + #[doc = concat!("use ndarray::hpc::amx_ops::{amx_features, ldtilecfg, tilerelease, tilezero, ", stringify!($name), "};")] + /// use ndarray::simd_amx::amx_tile_available; + /// + #[doc = concat!("if amx_tile_available() && amx_features().", stringify!($tier), " {")] + /// let cfg = TileConfig::for_dpbusd(64); + /// // SAFETY: tile permission held (checked above), the config covers + /// // tiles 0-2 with compatible shapes, and the operands are distinct. + /// unsafe { + /// ldtilecfg(cfg.data.as_ptr()); + /// tilezero::<0>(); + #[doc = concat!(" ", stringify!($name), "::<0, 2, 1>();")] + /// tilerelease(); + /// } + /// } + /// ``` + #[inline(always)] + pub unsafe fn $name() { + const { + assert!(D < 8 && S1 < 8 && S2 < 8); + assert!(D != S1 && D != S2 && S1 != S2, "tile operands must be distinct (#UD otherwise)"); + } + asm!(concat!($mn, " tmm{d}, tmm{a}, tmm{b}"), d = const D, a = const S1, b = const S2, options(nostack, nomem)); + } + }; +} + +tdp3!( + /// `TDPBSSD` — signed i8 × signed i8 → i32 (AMX-INT8). + tdpbssd, "tdpbssd", int8 +); +tdp3!( + /// `TDPBSUD` — signed i8 (S1) × unsigned u8 (S2) → i32 (AMX-INT8). + tdpbsud, "tdpbsud", int8 +); +tdp3!( + /// `TDPBUSD` — unsigned u8 (S1) × signed i8 (S2) → i32 (AMX-INT8). The + /// kernel's op: [`tdpbusd::<0, 2, 1>`] is the validated `C4 E2 71 5E C2`. + tdpbusd, "tdpbusd", int8 +); +tdp3!( + /// `TDPBUUD` — unsigned u8 × unsigned u8 → i32 (AMX-INT8). + tdpbuud, "tdpbuud", int8 +); +tdp3!( + /// `TDPBF16PS` — bf16 × bf16 → f32 (AMX-BF16). [`tdpbf16ps::<0, 2, 1>`] is + /// the validated `C4 E2 72 5C C2`. + tdpbf16ps, "tdpbf16ps", bf16 +); +tdp3!( + /// `TDPFP16PS` — fp16 × fp16 → f32 (AMX-FP16, Granite Rapids). + /// Assembler-verified only. + tdpfp16ps, "tdpfp16ps", fp16 +); +tdp3!( + /// `TCMMIMFP16PS` — imaginary part of a complex fp16 matrix product → f32 + /// (AMX-COMPLEX). Assembler-verified only. + tcmmimfp16ps, "tcmmimfp16ps", complex +); +tdp3!( + /// `TCMMRLFP16PS` — real part of a complex fp16 matrix product → f32 + /// (AMX-COMPLEX). Assembler-verified only. + tcmmrlfp16ps, "tcmmrlfp16ps", complex +); +tdp3!( + /// `TDPBF8PS` — E5M2 × E5M2 → f32 (AMX-FP8, Diamond Rapids). + /// Assembler-verified only. + tdpbf8ps, "tdpbf8ps", fp8 +); +tdp3!( + /// `TDPBHF8PS` — E5M2 (S1) × E4M3 (S2) → f32 (AMX-FP8). Assembler-verified only. + tdpbhf8ps, "tdpbhf8ps", fp8 +); +tdp3!( + /// `TDPHBF8PS` — E4M3 (S1) × E5M2 (S2) → f32 (AMX-FP8). Assembler-verified only. + tdphbf8ps, "tdphbf8ps", fp8 +); +tdp3!( + /// `TDPHF8PS` — E4M3 × E4M3 → f32 (AMX-FP8). Assembler-verified only. + tdphf8ps, "tdphf8ps", fp8 +); +/// `TMMULTF32PS` — tf32 × tf32 → f32 (AMX-TF32). CLAIMED — no host has +/// executed it. +/// +/// Emitted as raw bytes, not a mnemonic: LLVM `main` dropped `amx-tf32`, and +/// nightly's LLVM 23 rejects `tmmultf32ps` while stable's 22.1.8 still +/// assembles it. The encoding is fixed by the ISA — `C4 E2 48 ` +/// with `vex = (!S2 & 0xF) << 3 | 0b01` (W0, vvvv = S2 inverted, L0, pp=66) +/// and `modrm = 0xC0 | D << 3 | S1` — and reproduces the same byte table the +/// mnemonic form did (`C4 E2 69 48 C1` for tiles 0, 1, 2), which the +/// `extended_tiers_assemble_to_their_llvm_encodings` test pins. +/// +/// `D += S1 · S2`; S1 is the plain M×K operand (ModRM.rm), S2 the +/// VNNI-packed K×N operand (VEX.vvvv). The three tiles must be distinct — +/// enforced at compile time. +/// +/// # Safety +/// Tiles configured with compatible shapes, +/// [`crate::simd_amx::amx_tile_available`] true, and the host must report +/// [`AmxFeatures::tf32`]. The expected bytes were derived by LLVM 22.1.8's +/// assembler from the mnemonic (commit 9ebd2c5, green on stable) before the +/// wrapper switched to raw bytes; the encoding test pins both the `(0, 1, 2)` +/// and the `(0, 2, 1)` instantiation to that origin. +/// +/// # Examples +/// +/// ```rust,no_run +/// use ndarray::hpc::amx_matmul::TileConfig; +/// use ndarray::hpc::amx_ops::{amx_features, ldtilecfg, tilerelease, tilezero, tmmultf32ps}; +/// use ndarray::simd_amx::amx_tile_available; +/// +/// if amx_tile_available() && amx_features().tf32 { +/// let cfg = TileConfig::for_dpbusd(64); +/// // SAFETY: tile permission held, TF32 advertised, config covers tiles +/// // 0-2, operands distinct. +/// unsafe { +/// ldtilecfg(cfg.data.as_ptr()); +/// tilezero::<0>(); +/// tmmultf32ps::<0, 2, 1>(); +/// tilerelease(); +/// } +/// } +/// ``` +#[inline(always)] +pub unsafe fn tmmultf32ps() { + const { + assert!(D < 8 && S1 < 8 && S2 < 8); + assert!(D != S1 && D != S2 && S1 != S2, "tile operands must be distinct (#UD otherwise)"); + } + asm!( + ".byte 0xC4, 0xE2, {vex}, 0x48, {modrm}", + vex = const ((!S2 & 0x0F) << 3) | 0x01, + modrm = const 0xC0 | (D << 3) | S1, + options(nostack, nomem) + ); +} + +// ── AMX-AVX512 (Diamond Rapids): tile row → zmm ───────────────────────────── +// +// These need a zmm operand, which `asm!` only accepts when `avx512f` is a +// compile-time target feature, so they exist under the v4/native configs +// only. Same compile-time selection as everything else in this crate — no +// `#[target_feature]`, no runtime dispatch. + +#[cfg(target_feature = "avx512f")] +macro_rules! tile_row_to_zmm { + ($(#[$m:meta])* $name:ident, $name_imm:ident, $mn:literal, $ty:ty) => { + $(#[$m])* + /// + /// Register-row form: `row` selects the tile row at run time. + /// + /// # Safety + /// Tile `T` configured and holding data, + /// [`crate::simd_amx::amx_tile_available`] true, and the host must + /// report [`AmxFeatures::avx512`]. + /// + /// # Examples + /// + /// `ignore`d rather than `no_run` because this function exists only + /// when `avx512f` is a compile-time target feature (v4 / native + /// builds); a v3 doctest build would not find it. + /// + /// ```rust,ignore + /// use ndarray::hpc::amx_matmul::TileConfig; + #[doc = concat!("use ndarray::hpc::amx_ops::{amx_features, ldtilecfg, tilerelease, tilezero, ", stringify!($name), "};")] + /// use ndarray::simd_amx::amx_tile_available; + /// + /// if amx_tile_available() && amx_features().avx512 { + /// let cfg = TileConfig::for_dpbusd(64); + /// // SAFETY: tile permission held, AMX-AVX512 advertised, tile 0 + /// // configured and zeroed before its row is read. + /// unsafe { + /// ldtilecfg(cfg.data.as_ptr()); + /// tilezero::<0>(); + #[doc = concat!(" let _row0 = ", stringify!($name), "::<0>(0);")] + /// tilerelease(); + /// } + /// } + /// ``` + #[inline(always)] + pub unsafe fn $name(row: u32) -> $ty { + const { assert!(T < 8) } + let out: $ty; + asm!(concat!($mn, " {o}, tmm{t}, {r:e}"), o = out(zmm_reg) out, t = const T, r = in(reg) row, options(nostack, nomem)); + out + } + $(#[$m])* + /// + /// Immediate-row form: `ROW` is a compile-time constant. + /// + /// # Safety + /// As the register-row form. + /// + /// # Examples + /// + /// `ignore`d for the same reason as the register-row form (the + /// function exists only under a compile-time `avx512f`). + /// + /// ```rust,ignore + /// use ndarray::hpc::amx_matmul::TileConfig; + #[doc = concat!("use ndarray::hpc::amx_ops::{amx_features, ldtilecfg, tilerelease, tilezero, ", stringify!($name_imm), "};")] + /// use ndarray::simd_amx::amx_tile_available; + /// + /// if amx_tile_available() && amx_features().avx512 { + /// let cfg = TileConfig::for_dpbusd(64); + /// // SAFETY: tile permission held, AMX-AVX512 advertised, tile 0 + /// // configured and zeroed; ROW 3 < 16 configured rows. + /// unsafe { + /// ldtilecfg(cfg.data.as_ptr()); + /// tilezero::<0>(); + #[doc = concat!(" let _row3 = ", stringify!($name_imm), "::<0, 3>();")] + /// tilerelease(); + /// } + /// } + /// ``` + #[inline(always)] + pub unsafe fn $name_imm() -> $ty { + const { assert!(T < 8 && ROW < 16) } + let out: $ty; + asm!(concat!($mn, " {o}, tmm{t}, {r}"), o = out(zmm_reg) out, t = const T, r = const ROW, options(nostack, nomem)); + out + } + }; +} + +#[cfg(target_feature = "avx512f")] +tile_row_to_zmm!( + /// `TCVTROWD2PS` — one tile row of 16 × i32 converted to 16 × f32. + /// Assembler-verified only. + tcvtrowd2ps, tcvtrowd2ps_imm, "tcvtrowd2ps", core::arch::x86_64::__m512 +); +#[cfg(target_feature = "avx512f")] +tile_row_to_zmm!( + /// `TCVTROWPS2PHH` — tile row of f32 → fp16, high halves. Assembler-verified only. + tcvtrowps2phh, tcvtrowps2phh_imm, "tcvtrowps2phh", core::arch::x86_64::__m512i +); +#[cfg(target_feature = "avx512f")] +tile_row_to_zmm!( + /// `TCVTROWPS2PHL` — tile row of f32 → fp16, low halves. Assembler-verified only. + tcvtrowps2phl, tcvtrowps2phl_imm, "tcvtrowps2phl", core::arch::x86_64::__m512i +); +#[cfg(target_feature = "avx512f")] +tile_row_to_zmm!( + /// `TCVTROWPS2BF16H` — tile row of f32 → bf16, high halves. Assembler-verified only. + tcvtrowps2bf16h, tcvtrowps2bf16h_imm, "tcvtrowps2bf16h", core::arch::x86_64::__m512i +); +#[cfg(target_feature = "avx512f")] +tile_row_to_zmm!( + /// `TCVTROWPS2BF16L` — tile row of f32 → bf16, low halves. Assembler-verified only. + tcvtrowps2bf16l, tcvtrowps2bf16l_imm, "tcvtrowps2bf16l", core::arch::x86_64::__m512i +); +#[cfg(target_feature = "avx512f")] +tile_row_to_zmm!( + /// `TILEMOVROW` — one 64-byte tile row moved into a zmm unchanged. + /// Assembler-verified only. + tilemovrow, tilemovrow_imm, "tilemovrow", core::arch::x86_64::__m512i +); + +// ── Per-tier detection ────────────────────────────────────────────────────── + +/// Which AMX tiers this CPU advertises, per LLVM `Host.cpp`'s bit positions. +/// +/// Silicon bits only — the gate for "may I execute a tile op" is +/// [`crate::simd_amx::amx_tile_available`] (OS XSAVE state + `arch_prctl` +/// permission); [`super::amx_matmul::amx_available`] adds only the INT8 bit +/// and gates the INT8 ops. This struct answers "which ops exist once I may". +#[derive(Copy, Clone, Debug, PartialEq, Eq, Default)] +pub struct AmxFeatures { + /// AMX-TILE (7.0:EDX[24]). + pub tile: bool, + /// AMX-INT8 (7.0:EDX[25]). + pub int8: bool, + /// AMX-BF16 (7.0:EDX[22]). + pub bf16: bool, + /// AMX-FP16 (7.1:EAX[21]). + pub fp16: bool, + /// AMX-COMPLEX (7.1:EDX[8]). + pub complex: bool, + /// AMX-FP8 (1E.1:EAX[4]). + pub fp8: bool, + /// AMX-TF32 (1E.1:EAX[6]; the bit LLVM used before dropping the feature). + pub tf32: bool, + /// AMX-AVX512 (1E.1:EAX[7]). + pub avx512: bool, + /// AMX-MOVRS (1E.1:EAX[8]). + pub movrs: bool, +} + +fn detect_amx_features() -> AmxFeatures { + use core::arch::x86_64::{__cpuid, __cpuid_count, CpuidResult}; + let max_leaf = __cpuid(0).eax; + // An out-of-range basic leaf may return the HIGHEST basic leaf's data, so + // every leaf is guarded by `max_leaf` and an unavailable one reads as all + // zero — never as leaf 0 (vendor string + max leaf), whose bits are not + // feature bits either. + let zero = || CpuidResult { + eax: 0, + ebx: 0, + ecx: 0, + edx: 0, + }; + let l7_0 = if max_leaf >= 7 { __cpuid_count(7, 0) } else { zero() }; + let l7_1 = if max_leaf >= 7 && l7_0.eax >= 1 { + __cpuid_count(7, 1) + } else { + zero() + }; + let l1e_1 = if max_leaf >= 0x1e { + __cpuid_count(0x1e, 1) + } else { + zero() + }; + let bit = |v: u32, b: u32| (v >> b) & 1 == 1; + AmxFeatures { + tile: bit(l7_0.edx, 24), + int8: bit(l7_0.edx, 25), + bf16: bit(l7_0.edx, 22), + fp16: bit(l7_1.eax, 21), + complex: bit(l7_1.edx, 8), + fp8: max_leaf >= 0x1e && bit(l1e_1.eax, 4), + tf32: max_leaf >= 0x1e && bit(l1e_1.eax, 6), + avx512: max_leaf >= 0x1e && bit(l1e_1.eax, 7), + movrs: max_leaf >= 0x1e && bit(l1e_1.eax, 8), + } +} + +static AMX_FEATURES: std::sync::LazyLock = std::sync::LazyLock::new(detect_amx_features); + +/// The advertised AMX tiers, cached (CPUID is a serializing instruction; once +/// is enough). +/// +/// # Examples +/// +/// Runs on any x86_64 host — it only reads CPUID: +/// +/// ``` +/// use ndarray::hpc::amx_ops::amx_features; +/// let f = amx_features(); +/// println!("AMX tiers: tile={} int8={} bf16={} fp16={}", f.tile, f.int8, f.bf16, f.fp16); +/// if f.tile && f.int8 { +/// println!("the int8 GEMM tier exists on this silicon (OS gates still apply)"); +/// } +/// ``` +pub fn amx_features() -> AmxFeatures { + *AMX_FEATURES +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Everything that reads `/proc/self/exe` — Linux only, as its own doc + /// says; on another x86_64 OS these tests do not exist rather than fail + /// at the `expect`. The CPUID consistency test below stays OS-agnostic. + #[cfg(target_os = "linux")] + mod encodings { + use super::super::*; + + /// Read the machine code of a monomorphized op out of the OBJECT FILE — + /// the test binary itself, `/proc/self/exe` — via its ELF `.symtab`: the + /// symbol's `st_value`/`st_size` are the linker's own statement of where + /// the wrapper starts and how long it is, so the extent is validated by + /// the producer of the bytes, never inferred from a function pointer, a + /// fixed window, or a `ret`-byte heuristic (0xC3 can sit inside another + /// instruction's immediate). No executable memory is dereferenced at all. + /// Each wrapper carries an `export_name` so it can be found by name, and + /// `#[inline(never)]` so the op's bytes are its own symbol's bytes. Runs + /// on ANY x86_64 Linux host — it inspects encodings, never executes a + /// tile op — so the `.byte` tables in `amx_matmul` and the mnemonics here + /// are pinned to each other by CI, not by an EMR box. Requires an + /// unstripped test binary (cargo's default for every test profile). + fn symbol_bytes(name: &str) -> Vec { + // Read the binary once per process, not once per assertion (each + // lookup walks the whole `.symtab`; the file is tens of MiB). + static EXE: std::sync::OnceLock> = std::sync::OnceLock::new(); + let exe = EXE.get_or_init(|| std::fs::read("/proc/self/exe").expect("read /proc/self/exe")); + let u16_at = |o: usize| u16::from_le_bytes([exe[o], exe[o + 1]]); + let u32_at = |o: usize| u32::from_le_bytes(exe[o..o + 4].try_into().expect("4 bytes")); + let u64_at = |o: usize| u64::from_le_bytes(exe[o..o + 8].try_into().expect("8 bytes")); + assert_eq!(&exe[..4], b"\x7fELF", "test binary is ELF"); + assert_eq!(exe[4], 2, "ELF64"); + let shoff = u64_at(0x28) as usize; + let shentsize = u16_at(0x3a) as usize; + let shnum = u16_at(0x3c) as usize; + // (sh_type, sh_addr, sh_offset, sh_size, sh_link, sh_entsize) + let section = |i: usize| { + let b = shoff + i * shentsize; + ( + u32_at(b + 4), + u64_at(b + 0x10), + u64_at(b + 0x18), + u64_at(b + 0x20), + u32_at(b + 0x28), + u64_at(b + 0x38), + ) + }; + const SHT_SYMTAB: u32 = 2; + let symtab = (0..shnum) + .map(section) + .find(|s| s.0 == SHT_SYMTAB) + .expect("test binary carries .symtab — do not strip test binaries"); + let strtab = section(symtab.4 as usize); + let entsize = symtab.5 as usize; + assert_eq!(entsize, 24, "Elf64_Sym"); + for i in 0..(symtab.3 as usize / entsize) { + let b = symtab.2 as usize + i * entsize; + let name_off = strtab.2 as usize + u32_at(b) as usize; + let name_len = exe[name_off..] + .iter() + .position(|&c| c == 0) + .expect("NUL-terminated symbol name"); + if &exe[name_off..name_off + name_len] != name.as_bytes() { + continue; + } + // ORDER MATTERS: the section lookup below must stay AFTER the name + // match. Hundreds of symbols in this binary carry a special + // `st_shndx` (`SHN_ABS` 0xfff1 for every `STT_FILE`, `SHN_UNDEF`, + // …) that is not a section index at all; `section()` on one of + // them indexes past the section table. A matched probe is always + // a real `FUNC` in `.text`, so only the match may reach it. + let st_shndx = u16_at(b + 6) as usize; + let st_value = u64_at(b + 8); + let st_size = u64_at(b + 16) as usize; + assert!(st_size > 0, "{name}: symbol has no size"); + let sec = section(st_shndx); + let file_off = (st_value - sec.1 + sec.2) as usize; + return exe[file_off..file_off + st_size].to_vec(); + } + panic!("symbol {name} not found in .symtab"); + } + + /// A wrapper as (fn pointer, exported symbol name). The pointer is only + /// ever passed through `black_box` — it is never dereferenced — so that + /// the otherwise-unreferenced wrapper is actually codegen'd into the test + /// binary (an `export_name` alone does not keep a dead fn alive here; + /// measured: 0 probe symbols in `.symtab` without the reference). + macro_rules! probe { + ($w:ident) => { + ($w as unsafe fn(), concat!("ndarray_amx_probe_", stringify!($w))) + }; + } + + /// Does the wrapper's own symbol contain the exact encoding? Bounded by + /// the symbol's linker-recorded size, so a neighbouring wrapper's bytes + /// can neither fail a negative assertion nor pass a positive one. + fn contains((f, name): (unsafe fn(), &str), needle: &[u8]) -> bool { + std::hint::black_box(f as usize); + symbol_bytes(name) + .windows(needle.len()) + .any(|w| w == needle) + } + + /// `contains` with a per-byte mask, for ops whose encoding carries a + /// register the ALLOCATOR chooses: a memory operand's base/index land in + /// VEX byte 1's B/X bits and the SIB byte, a zmm destination in the + /// ModRM.reg field and EVEX R/R'. Those bits are masked OFF; the opcode, + /// prefix map, W/L/pp, the tile number and any immediate are matched + /// exactly. `(byte, mask)` pairs; a mask of `0xff` is an exact byte. + fn contains_masked((f, name): (unsafe fn(), &str), needle: &[(u8, u8)]) -> bool { + std::hint::black_box(f as usize); + symbol_bytes(name) + .windows(needle.len()) + .any(|w| w.iter().zip(needle).all(|(&b, &(e, m))| b & m == e & m)) + } + + #[inline(never)] + #[export_name = "ndarray_amx_probe_w_tilezero0"] + unsafe fn w_tilezero0() { + tilezero::<0>() + } + #[inline(never)] + #[export_name = "ndarray_amx_probe_w_tilezero7"] + unsafe fn w_tilezero7() { + tilezero::<7>() + } + #[inline(never)] + #[export_name = "ndarray_amx_probe_w_tilerelease"] + unsafe fn w_tilerelease() { + tilerelease() + } + #[inline(never)] + #[export_name = "ndarray_amx_probe_w_tdpbusd_021"] + unsafe fn w_tdpbusd_021() { + tdpbusd::<0, 2, 1>() + } + #[inline(never)] + #[export_name = "ndarray_amx_probe_w_tdpbf16ps_021"] + unsafe fn w_tdpbf16ps_021() { + tdpbf16ps::<0, 2, 1>() + } + #[inline(never)] + #[export_name = "ndarray_amx_probe_w_tdpbusd_012"] + unsafe fn w_tdpbusd_012() { + tdpbusd::<0, 1, 2>() + } + #[inline(never)] + #[export_name = "ndarray_amx_probe_w_tdpbssd_012"] + unsafe fn w_tdpbssd_012() { + tdpbssd::<0, 1, 2>() + } + #[inline(never)] + #[export_name = "ndarray_amx_probe_w_tdpfp16ps_012"] + unsafe fn w_tdpfp16ps_012() { + tdpfp16ps::<0, 1, 2>() + } + #[inline(never)] + #[export_name = "ndarray_amx_probe_w_tcmmimfp16ps_012"] + unsafe fn w_tcmmimfp16ps_012() { + tcmmimfp16ps::<0, 1, 2>() + } + #[inline(never)] + #[export_name = "ndarray_amx_probe_w_tdpbf8ps_012"] + unsafe fn w_tdpbf8ps_012() { + tdpbf8ps::<0, 1, 2>() + } + #[inline(never)] + #[export_name = "ndarray_amx_probe_w_tdphf8ps_012"] + unsafe fn w_tdphf8ps_012() { + tdphf8ps::<0, 1, 2>() + } + #[inline(never)] + #[export_name = "ndarray_amx_probe_w_tmmultf32ps_012"] + unsafe fn w_tmmultf32ps_012() { + tmmultf32ps::<0, 1, 2>() + } + #[inline(never)] + #[export_name = "ndarray_amx_probe_w_tmmultf32ps_021"] + unsafe fn w_tmmultf32ps_021() { + tmmultf32ps::<0, 2, 1>() + } + #[inline(never)] + #[export_name = "ndarray_amx_probe_w_tdpbsud_012"] + unsafe fn w_tdpbsud_012() { + tdpbsud::<0, 1, 2>() + } + #[inline(never)] + #[export_name = "ndarray_amx_probe_w_tdpbuud_012"] + unsafe fn w_tdpbuud_012() { + tdpbuud::<0, 1, 2>() + } + #[inline(never)] + #[export_name = "ndarray_amx_probe_w_tcmmrlfp16ps_012"] + unsafe fn w_tcmmrlfp16ps_012() { + tcmmrlfp16ps::<0, 1, 2>() + } + #[inline(never)] + #[export_name = "ndarray_amx_probe_w_tdpbhf8ps_012"] + unsafe fn w_tdpbhf8ps_012() { + tdpbhf8ps::<0, 1, 2>() + } + #[inline(never)] + #[export_name = "ndarray_amx_probe_w_tdphbf8ps_012"] + unsafe fn w_tdphbf8ps_012() { + tdphbf8ps::<0, 1, 2>() + } + + // Memory-operand ops. The pointer/stride come from `black_box` so the + // allocator picks the registers; the masked matcher ignores them. + static SCRATCH: [u8; 64] = [0u8; 64]; + #[inline(never)] + #[export_name = "ndarray_amx_probe_w_ldtilecfg"] + unsafe fn w_ldtilecfg() { + ldtilecfg(std::hint::black_box(SCRATCH.as_ptr())) + } + #[inline(never)] + #[export_name = "ndarray_amx_probe_w_sttilecfg"] + unsafe fn w_sttilecfg() { + sttilecfg(std::hint::black_box(SCRATCH.as_ptr() as *mut u8)) + } + #[inline(never)] + #[export_name = "ndarray_amx_probe_w_tileloadd2"] + unsafe fn w_tileloadd2() { + tileloadd::<2>(std::hint::black_box(SCRATCH.as_ptr()), std::hint::black_box(64)) + } + #[inline(never)] + #[export_name = "ndarray_amx_probe_w_tileloaddt1_2"] + unsafe fn w_tileloaddt1_2() { + tileloaddt1::<2>(std::hint::black_box(SCRATCH.as_ptr()), std::hint::black_box(64)) + } + #[inline(never)] + #[export_name = "ndarray_amx_probe_w_tilestored0"] + unsafe fn w_tilestored0() { + tilestored::<0>(std::hint::black_box(SCRATCH.as_ptr() as *mut u8), std::hint::black_box(64)) + } + #[inline(never)] + #[export_name = "ndarray_amx_probe_w_tileloaddrs2"] + unsafe fn w_tileloaddrs2() { + tileloaddrs::<2>(std::hint::black_box(SCRATCH.as_ptr()), std::hint::black_box(64)) + } + #[inline(never)] + #[export_name = "ndarray_amx_probe_w_tileloaddrst1_2"] + unsafe fn w_tileloaddrst1_2() { + tileloaddrst1::<2>(std::hint::black_box(SCRATCH.as_ptr()), std::hint::black_box(64)) + } + #[cfg(target_feature = "avx512f")] + #[inline(never)] + #[export_name = "ndarray_amx_probe_w_tcvtrowd2ps_imm_1_3"] + unsafe fn w_tcvtrowd2ps_imm_1_3() { + let _ = std::hint::black_box(tcvtrowd2ps_imm::<1, 3>()); + } + #[cfg(target_feature = "avx512f")] + #[inline(never)] + #[export_name = "ndarray_amx_probe_w_tilemovrow_imm_1_5"] + unsafe fn w_tilemovrow_imm_1_5() { + let _ = std::hint::black_box(tilemovrow_imm::<1, 5>()); + } + + /// The mnemonic path must reproduce `amx_matmul`'s validated `.byte` + /// table byte for byte — these are the sequences measured on Emerald + /// Rapids (`amx-enablement-and-kernel.md` §5). + #[test] + fn mnemonics_reproduce_the_validated_byte_table() { + assert!(contains(probe!(w_tilezero0), &[0xc4, 0xe2, 0x7b, 0x49, 0xc0]), "TILEZERO tmm0"); + assert!(contains(probe!(w_tilerelease), &[0xc4, 0xe2, 0x78, 0x49, 0xc0]), "TILERELEASE"); + assert!( + contains(probe!(w_tdpbusd_021), &[0xc4, 0xe2, 0x71, 0x5e, 0xc2]), + "TDPBUSD tmm0, tmm2, tmm1 == table C4 E2 71 5E C2" + ); + assert!( + contains(probe!(w_tdpbf16ps_021), &[0xc4, 0xe2, 0x72, 0x5c, 0xc2]), + "TDPBF16PS tmm0, tmm2, tmm1 == table C4 E2 72 5C C2" + ); + } + + /// The operand convention, stated as bytes: swapping S1/S2 swaps + /// ModRM.rm and VEX.vvvv, nothing else. A body that silently reordered + /// the operands (the "mirror" the gotchas warn about) would fail one half. + #[test] + fn operand_order_is_intel_order_rm_then_vvvv() { + assert!( + contains(probe!(w_tdpbusd_012), &[0xc4, 0xe2, 0x69, 0x5e, 0xc1]), + "tdpbusd tmm0,tmm1,tmm2 → rm=1 vvvv=2" + ); + assert!( + contains(probe!(w_tdpbusd_021), &[0xc4, 0xe2, 0x71, 0x5e, 0xc2]), + "tdpbusd tmm0,tmm2,tmm1 → rm=2 vvvv=1" + ); + assert!(!contains(probe!(w_tdpbusd_012), &[0xc4, 0xe2, 0x71, 0x5e, 0xc2])); + } + + /// Beyond the GEMM tier: the bytes LLVM 22.1.8 emits for the mnemonics + /// that have never executed here (assembler-verified, per the module doc). + #[test] + fn extended_tiers_assemble_to_their_llvm_encodings() { + assert!(contains(probe!(w_tilezero7), &[0xc4, 0xe2, 0x7b, 0x49, 0xf8]), "TILEZERO tmm7"); + assert!(contains(probe!(w_tdpbssd_012), &[0xc4, 0xe2, 0x6b, 0x5e, 0xc1]), "TDPBSSD (F2 prefix)"); + assert!(contains(probe!(w_tdpfp16ps_012), &[0xc4, 0xe2, 0x6b, 0x5c, 0xc1]), "TDPFP16PS = 5C with F2"); + assert!(contains(probe!(w_tcmmimfp16ps_012), &[0xc4, 0xe2, 0x69, 0x6c, 0xc1]), "TCMMIMFP16PS = 6C with 66"); + assert!(contains(probe!(w_tdpbf8ps_012), &[0xc4, 0xe5, 0x68, 0xfd, 0xc1]), "TDPBF8PS = map5 FD, no prefix"); + assert!(contains(probe!(w_tdphf8ps_012), &[0xc4, 0xe5, 0x69, 0xfd, 0xc1]), "TDPHF8PS = map5 FD, 66"); + assert!(contains(probe!(w_tmmultf32ps_012), &[0xc4, 0xe2, 0x69, 0x48, 0xc1]), "TMMULTF32PS = 48 with 66"); + assert!( + contains(probe!(w_tmmultf32ps_021), &[0xc4, 0xe2, 0x71, 0x48, 0xc2]), + "TMMULTF32PS (0,2,1): vvvv=~1 → 71, rm=2 → C2" + ); + assert!(contains(probe!(w_tdpbsud_012), &[0xc4, 0xe2, 0x6a, 0x5e, 0xc1]), "TDPBSUD (F3 prefix)"); + assert!(contains(probe!(w_tdpbuud_012), &[0xc4, 0xe2, 0x68, 0x5e, 0xc1]), "TDPBUUD (no prefix)"); + assert!(contains(probe!(w_tcmmrlfp16ps_012), &[0xc4, 0xe2, 0x68, 0x6c, 0xc1]), "TCMMRLFP16PS = 6C, NP"); + assert!(contains(probe!(w_tdpbhf8ps_012), &[0xc4, 0xe5, 0x6b, 0xfd, 0xc1]), "TDPBHF8PS = map5 FD, F2"); + assert!(contains(probe!(w_tdphbf8ps_012), &[0xc4, 0xe5, 0x6a, 0xfd, 0xc1]), "TDPHBF8PS = map5 FD, F3"); + } + + /// The memory-operand ops: opcode, map, prefix and TILE NUMBER pinned; + /// the base/index registers (VEX B/X bits, SIB) are the allocator's and + /// are masked. `E` = exact byte, `B1` = VEX byte 1 with R/X/B masked, + /// `REG(t)` = ModRM with only the reg field (the tile) compared. + #[test] + fn memory_operand_ops_pin_opcode_prefix_and_tile() { + const E: u8 = 0xff; + const B1: (u8, u8) = (0xe2, 0x1f); + const fn reg(t: u8) -> (u8, u8) { + (t << 3, 0x38) + } + // LDTILECFG: VEX.128.NP.0F38.W0 49 /0 + assert!(contains_masked(probe!(w_ldtilecfg), &[(0xc4, E), B1, (0x78, E), (0x49, E), reg(0)]), "LDTILECFG"); + // STTILECFG: VEX.128.66.0F38.W0 49 /0 + assert!(contains_masked(probe!(w_sttilecfg), &[(0xc4, E), B1, (0x79, E), (0x49, E), reg(0)]), "STTILECFG"); + // TILELOADD tmm2: VEX.128.F2.0F38.W0 4B /r + assert!(contains_masked(probe!(w_tileloadd2), &[(0xc4, E), B1, (0x7b, E), (0x4b, E), reg(2)]), "TILELOADD"); + // TILELOADDT1 tmm2: 66 prefix + assert!( + contains_masked(probe!(w_tileloaddt1_2), &[(0xc4, E), B1, (0x79, E), (0x4b, E), reg(2)]), + "TILELOADDT1" + ); + // TILESTORED tmm0: F3 prefix + assert!( + contains_masked(probe!(w_tilestored0), &[(0xc4, E), B1, (0x7a, E), (0x4b, E), reg(0)]), + "TILESTORED" + ); + // TILELOADDRS tmm2 / TILELOADDRST1 tmm2: opcode 4A, F2 / 66 + assert!( + contains_masked(probe!(w_tileloaddrs2), &[(0xc4, E), B1, (0x7b, E), (0x4a, E), reg(2)]), + "TILELOADDRS" + ); + assert!( + contains_masked(probe!(w_tileloaddrst1_2), &[(0xc4, E), B1, (0x79, E), (0x4a, E), reg(2)]), + "TILELOADDRST1" + ); + // A wrong tile number must fail: tmm2's reg field is not tmm3's. + assert!(!contains_masked(probe!(w_tileloadd2), &[(0xc4, E), B1, (0x7b, E), (0x4b, E), reg(3)])); + } + + /// AMX-AVX512 row ops, immediate-row forms (`avx512f` builds only): EVEX + /// map, prefix, opcode, tile and immediate pinned; the zmm destination + /// (EVEX R/R' in byte 1, ModRM.reg) is the allocator's and is masked. + #[cfg(target_feature = "avx512f")] + #[test] + fn avx512_row_ops_assemble_to_their_llvm_encodings() { + const E: u8 = 0xff; + // TCVTROWD2PS zmm, tmm1, imm8 = EVEX 62 F3 7E 48 07 /r ib + assert!( + contains_masked( + probe!(w_tcvtrowd2ps_imm_1_3), + &[(0x62, E), (0xf3, 0x6f), (0x7e, E), (0x48, E), (0x07, E), (0xc1, 0xc7), (0x03, E)] + ), + "TCVTROWD2PS imm" + ); + // TILEMOVROW zmm, tmm1, imm8 = EVEX 62 F3 7D 48 07 /r ib + assert!( + contains_masked( + probe!(w_tilemovrow_imm_1_5), + &[(0x62, E), (0xf3, 0x6f), (0x7d, E), (0x48, E), (0x07, E), (0xc1, 0xc7), (0x05, E)] + ), + "TILEMOVROW imm" + ); + } + } + + /// A consistency re-derivation, and honest about its reach: on a host + /// without AMX (every GitHub runner) all six bits are `false` on both + /// sides and the equalities are `false == false` — a wrong bit POSITION + /// in `detect_amx_features` is only caught on AMX silicon. The + /// encoding tests carry the real weight; this one prints what it saw. + #[test] + fn feature_bits_are_consistent_with_the_legacy_detector() { + let f = amx_features(); + let l7 = core::arch::x86_64::__cpuid_count(7, 0); + assert_eq!(f.tile, (l7.edx >> 24) & 1 == 1); + assert_eq!(f.int8, (l7.edx >> 25) & 1 == 1); + assert_eq!(f.bf16, (l7.edx >> 22) & 1 == 1); + eprintln!("amx feature bits on this host: {f:?} (non-AMX host ⇒ this test is a no-op)"); + // "An extended tier implies TILE" is a silicon expectation, not a + // guarantee — a hypervisor masks CPUID bits arbitrarily — so it is + // observed, not asserted. + for (name, ext) in [ + ("fp16", f.fp16), + ("complex", f.complex), + ("fp8", f.fp8), + ("tf32", f.tf32), + ("avx512", f.avx512), + ("movrs", f.movrs), + ] { + if ext && !f.tile { + eprintln!("note: CPUID advertises AMX-{name} without AMX-TILE (hypervisor mask?)"); + } + } + } +} diff --git a/src/hpc/mod.rs b/src/hpc/mod.rs index 4d7df139..074921b6 100644 --- a/src/hpc/mod.rs +++ b/src/hpc/mod.rs @@ -89,6 +89,11 @@ pub use crate::heel_f64x8; #[cfg(target_arch = "x86_64")] #[allow(missing_docs)] pub mod amx_matmul; +/// The full AMX mnemonic surface (INT8 / BF16 / FP16 / COMPLEX / FP8 / TF32 / +/// MOVRS / AVX512 row ops) with `const` tile operands, plus per-tier CPUID +/// detection — see the module doc for what has executed vs. only assembled. +#[cfg(target_arch = "x86_64")] +pub mod amx_ops; #[cfg(target_arch = "x86_64")] pub mod bf16_tile_gemm; /// INT8 (`u8 × i8 → i32`) tile GEMM via AMX `TDPBUSD` — mirror of diff --git a/src/lib.rs b/src/lib.rs index 8f8147eb..0c1bb999 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -452,6 +452,13 @@ pub mod simd_soa; #[allow(missing_docs)] pub mod simd_int_ops; +/// Packed-bitmask predicates, mask algebra, and masked reductions — the +/// ergonomic masking layer between consumers and the compile-time-selected +/// backend (`eq_u32_to_mask`, `mask_ternlog`, `masked_sum_i32`, …). Owns +/// slice/tail/in-place ergonomics only; never an ISA. +#[cfg(feature = "std")] +pub mod simd_masking_ops; + /// Slice-level elementwise ops (f32/f64) built on the polyfill SIMD types. /// `add_f32`, `mul_f32`, `add_f32_inplace`, `scale_f32`, etc. /// Re-exported flat through `ndarray::simd::add_f32`. diff --git a/src/simd.rs b/src/simd.rs index fde15b91..ec0099c9 100644 --- a/src/simd.rs +++ b/src/simd.rs @@ -233,10 +233,11 @@ pub const PREFERRED_I16_LANES: usize = 16; // as soon as `nightly-simd` is on. #[cfg(feature = "nightly-simd")] pub use crate::simd_nightly::{ - f32x16, f32x8, f64x4, f64x8, i16x16, i16x32, i32x16, i32x8, i64x4, i64x8, i8x32, i8x64, u16x16, u16x32, u32x16, - u32x8, u64x4, u64x8, u8x32, u8x64, BF16x16, BF16x8, F16x16, F32Mask16, F32Mask8, F32x16, F32x8, F64Mask4, F64Mask8, - F64x4, F64x8, I16x16, I16x32, I32x16, I32x8, I64x4, I64x8, I8x32, I8x64, U16x16, U16x32, U32x16, U32x8, U64x4, - U64x8, U8x32, U8x64, + batch_packed_i4_16, f32x16, f32x8, f64x4, f64x8, i16x16, i16x32, i32x16, i32x8, i64x4, i64x8, i8x16, i8x32, i8x64, + palette_lookup_u8x8, prefetch_read_t0, prefetch_read_t1, prefetch_read_t2, u16x16, u16x32, u16x8, u32x16, u32x8, + u64x4, u64x8, u8x32, u8x64, u8x8, BF16x16, BF16x8, F16x16, F32Mask16, F32Mask8, F32x16, F32x8, F64Mask4, F64Mask8, + F64x4, F64x8, I16x16, I16x32, I32x16, I32x8, I64x4, I64x8, I8x16, I8x32, I8x64, U16x16, U16x32, U16x8, U32x16, + U32x8, U64x4, U64x8, U8x32, U8x64, U8x8, }; #[cfg(all(target_arch = "x86_64", target_feature = "avx512f", not(feature = "nightly-simd")))] @@ -316,16 +317,23 @@ pub use crate::simd_avx512::{f32_to_bf16_batch_rne, f32_to_bf16_scalar_rne}; pub use crate::simd_avx512::{BF16x16, BF16x8}; // AVX2 baseline arm — selected by the `x86-64-v3` cargo default. The -// predicate is `not(avx512f)` rather than `avx2 + not(avx512f)`: the -// inner intrinsics in `simd_avx2.rs` use per-function `#[target_feature -// (enable = "avx,avx2,fma")]` annotations, so the OPERATIONS gate -// themselves at the symbol level even when the consumer build target -// is x86-64 baseline. The struct-field types (`__m256` / `__m256i`) -// are core::arch declarations and don't require AVX/AVX2 at the type -// level — only execution does. Keeps GitHub CI green (it runs with -// `RUSTFLAGS="-D warnings"` env, which overrides our v3 config.toml, -// landing on x86-64 baseline → the previous tighter `avx2` predicate -// left no matching arm). +// predicate is `not(avx512f)` rather than `avx2 + not(avx512f)` so that +// an x86-64 baseline build (e.g. a `RUSTFLAGS` env that REPLACES the +// `.cargo/config.toml` target-cpu pin) still has a matching arm and +// COMPILES: the struct-field types (`__m256` / `__m256i`) are core::arch +// declarations that need no target feature at the type level. +// +// CORRECTED 2026-09-14: an earlier version of this comment claimed the +// inner intrinsics in `simd_avx2.rs` carry per-function +// `#[target_feature(enable = "avx,avx2,fma")]`. They do not, and by +// standing rule they must not — every `simd_{isa}.rs` file is one +// backend for one compile-time target, so a per-function feature gate is +// a second, contradictory selection mechanism. The intrinsic calls in +// `simd_avx2.rs` sit inside narrow `unsafe` blocks whose SAFETY +// precondition is the v3 baseline `.cargo/config.toml` pins for every +// x86_64 build; a baseline build compiles this arm but is not a supported +// execution target for it (it would SIGILL on the first `vp*` — the +// PR #170 failure mode the config pin exists to prevent). #[cfg(all( target_arch = "x86_64", not(target_feature = "avx512f"), @@ -387,10 +395,20 @@ pub use crate::simd_neon::{u16x8, U16x8}; // from simd_neon, not the scalar fallback, so it carries Add/BitXor/rotate_left. #[cfg(all(target_arch = "aarch64", not(feature = "nightly-simd")))] pub use crate::simd_neon::{u32x16, U32x16}; +// U64x8 + I32x16 — native `[uint64x2_t; 4]` / `[int32x4_t; 4]` fan-outs since +// 2026-09-13 (the five-flavour audit of #306: every bulk mask op and the whole +// signed-compare family ride these two types, and both used to resolve to the +// scalar backend here — the polyfill law wants a peer realisation per backend). +// The lowercase aliases travel WITH the types: `i32x16`/`u64x8` must name the +// same nominal type as `I32x16`/`U64x8` on every arm, so they come from the +// arm that owns the type (an alias left on the scalar list would silently +// split the facade into two types on this arch — product-engineer, PR #306). +#[cfg(all(target_arch = "aarch64", not(feature = "nightly-simd")))] +pub use crate::simd_neon::{i32x16, u64x8, I32x16, U64x8}; #[cfg(all(target_arch = "aarch64", not(feature = "nightly-simd")))] pub use scalar::{ - f32x8, f64x4, i32x16, i32x8, i64x4, i64x8, u16x16, u32x8, u64x4, u64x8, u8x64, F32x8, F64x4, I32x16, I32x8, I64x4, - I64x8, U16x16, U16x32, U32x8, U64x4, U64x8, U8x64, + f32x8, f64x4, i32x8, i64x4, i64x8, u16x16, u32x8, u64x4, u8x64, F32x8, F64x4, I32x8, I64x4, I64x8, U16x16, U16x32, + U32x8, U64x4, U8x64, }; // wasm32 + simd128: the native v128 float hot path (F32x16 / F64x8 + masks) @@ -401,16 +419,16 @@ pub use scalar::{ // so this arm is gated identically. #[cfg(all(target_arch = "wasm32", target_feature = "simd128", not(feature = "nightly-simd")))] pub use crate::simd_wasm::wasm32_simd::{ - f32x16, f64x8, i8x16, u32x16, F32Mask16, F32x16, F64Mask8, F64x8, I8x16, U32x16, + f32x16, f64x8, i32x16, i8x16, u32x16, u64x8, F32Mask16, F32x16, F64Mask8, F64x8, I32x16, I8x16, U32x16, U64x8, }; -// `u32x16`/`U32x16` now come from the native `wasm32_simd` arm above (the ARX -// lane the ChaCha20 backend rides), so they are dropped from this scalar list. +// `u32x16`/`U32x16`, `i32x16`/`I32x16` and `u64x8`/`U64x8` come from the +// native `wasm32_simd` arm above (the lowercase alias travels with its type — +// see the aarch64 note), so they are dropped from this scalar list. #[cfg(all(target_arch = "wasm32", target_feature = "simd128", not(feature = "nightly-simd")))] pub use scalar::{ - batch_packed_i4_16, f32x8, f64x4, i16x16, i16x32, i32x16, i32x8, i64x4, i64x8, i8x32, i8x64, palette_lookup_u8x8, - prefetch_read_t0, prefetch_read_t1, prefetch_read_t2, u16x16, u16x8, u32x8, u64x4, u64x8, u8x64, u8x8, F32x8, - F64x4, I16x16, I16x32, I32x16, I32x8, I64x4, I64x8, I8x32, I8x64, U16x16, U16x32, U16x8, U32x8, U64x4, U64x8, - U8x64, U8x8, + batch_packed_i4_16, f32x8, f64x4, i16x16, i16x32, i32x8, i64x4, i64x8, i8x32, i8x64, palette_lookup_u8x8, + prefetch_read_t0, prefetch_read_t1, prefetch_read_t2, u16x16, u16x8, u32x8, u64x4, u8x64, u8x8, F32x8, F64x4, + I16x16, I16x32, I32x8, I64x4, I64x8, I8x32, I8x64, U16x16, U16x32, U16x8, U32x8, U64x4, U8x64, U8x8, }; // Other non-x86 targets — wasm32 without simd128, riscv, etc.: full scalar @@ -584,6 +602,13 @@ pub mod ternlog { pub const AND2: i32 = 0xC0; /// `a | b | c` — union of three masks. pub const OR3: i32 = 0xFE; + /// `(a ^ b) & c` — the bits where `a` and `b` DIFFER, restricted to the + /// care set `c`; zero iff `a` matches `b` under care. The care-masked + /// (ternary) match kernel: `ternary_match_*_to_mask` tests this for zero. + pub const XOR_AND: i32 = 0x28; + /// `(a & b) | c` — two prerequisites, or an override. The immediate the + /// `lance-graph-duckmask` flagship `(A & B) | C` lowers to. + pub const AND2_OR: i32 = 0xEA; } pub use crate::hpc::bitwise::{hamming_distance_raw, popcount_raw}; @@ -633,16 +658,28 @@ pub use crate::hpc::quantized::{ // On all other targets (including avx512f-without-bf16, NEON, scalar) the // portable `simd_half::BF16x16` is the canonical 16-lane BF16 vector. -// Always re-export F16x16 + all slice-level ops (no naming conflict). +// Always re-export the slice-level ops (no naming conflict). #[cfg(feature = "std")] pub use crate::simd_half::{ add_bf16_inplace, add_f16_inplace, cast_bf16_to_f32_batch, cast_f16_to_f32_batch, cast_f32_to_bf16_batch, - cast_f32_to_f16_batch, mul_bf16_inplace, mul_f16_inplace, F16x16, + cast_f32_to_f16_batch, mul_bf16_inplace, mul_f16_inplace, }; -// Re-export portable BF16x16 only when the hardware-native avx512bf16 variant -// is NOT active (otherwise `simd_avx512::BF16x16` already occupies the name). -#[cfg(all(feature = "std", not(all(target_arch = "x86_64", target_feature = "avx512bf16"))))] +// The portable `simd_half::F16x16` yields the name to a backend that owns it: +// the `nightly-simd` arm above re-exports its own `F16x16`, so under that +// feature this re-export was a second definition (E0252 — one of the 25 +// nightly-rot errors fixed 2026-09-14). Same rule as `BF16x16` below. +#[cfg(all(feature = "std", not(feature = "nightly-simd")))] +pub use crate::simd_half::F16x16; + +// Re-export portable BF16x16 only when neither hardware-native variant owns the +// name (`simd_avx512::BF16x16` under avx512bf16; `simd_nightly::BF16x16` under +// `nightly-simd`). +#[cfg(all( + feature = "std", + not(feature = "nightly-simd"), + not(all(target_arch = "x86_64", target_feature = "avx512bf16")) +))] pub use crate::simd_half::BF16x16; // K-means + L2 distance @@ -718,19 +755,59 @@ pub use crate::hpc::bf16_tile_gemm::{ // silicon" from "AMX present but not OS-enabled" — both surface via `amx_report`. #[cfg(target_arch = "x86_64")] pub use crate::simd_amx::{amx_report, cpu_model, CpuModel}; - -// Packed-bitmask predicates + mask algebra — the columnar-selection lane. -// Slice-level siblings of `add_i8` / `dot_i8`, built on the lane-level -// `U32x16::eq_bitmask` / `I32x16::gt_bitmask` methods. Surfaced here because -// the W1a invariant is "all SIMD from `ndarray::simd`": a consumer that had to -// reach into `ndarray::simd_int_ops` (or worse, write its own compare-and-pack -// loop) would be a polyfill bypass. Bit order is normative and identical -// across all of them — element `i` at bit `i % 64` of word `i / 64`, trailing -// bits zero. See `src/simd_int_ops.rs` for the full statement. +// The tier-agnostic tile gate and the per-tier silicon bits: precondition #1 +// and #2 of every op in `hpc::amx_ops`, reachable through the facade so a +// consumer under the "all SIMD from `ndarray::simd`" rule can state them. +#[cfg(target_arch = "x86_64")] +pub use crate::hpc::amx_ops::{amx_features, AmxFeatures}; +#[cfg(target_arch = "x86_64")] +pub use crate::simd_amx::amx_tile_available; + +// Packed-bitmask predicates + mask algebra + masked reductions — the +// columnar-selection lane, owned by `simd_masking_ops.rs` (the ergonomic +// masking layer: slice/tail/in-place composition over the lane-level +// `U32x16::eq_bitmask` / `I32x16::gt_bitmask` / `U64x8::ternlog` methods, +// never an ISA). Surfaced here because the W1a invariant is "all SIMD from +// `ndarray::simd`": a consumer that reached into `ndarray::simd_masking_ops` +// directly (or worse, wrote its own compare-and-pack loop) would be a +// polyfill bypass. Bit order is normative and identical across all of them — +// element `i` at bit `i % 64` of word `i / 64`, trailing bits zero. See +// `src/simd_masking_ops.rs` for the full statement. #[cfg(feature = "std")] -pub use crate::simd_int_ops::{ - eq_u32_strided_to_mask, eq_u32_to_mask, gt_i32_to_mask, mask_and, mask_and_assign, mask_andnot, mask_andnot_assign, - mask_or, mask_or_assign, mask_ternlog, mask_ternlog_assign, masked_strided_group_sum, masked_sum_i32, +pub use crate::simd_masking_ops::{ + // 2026-09-13: the closed comparison family + complement/xor/any/all + care-masked + // register match + masked min/max + blend (lance-graph-duckmask, lgj-abi D-MRL-1a). + blend_i32, + eq_i32_to_mask, + eq_u32_strided_to_mask, + eq_u32_to_mask, + ge_i32_to_mask, + gt_i32_to_mask, + le_i32_to_mask, + lt_i32_to_mask, + mask_all, + mask_and, + mask_and_assign, + mask_andnot, + mask_andnot_assign, + mask_any, + mask_not, + mask_not_assign, + mask_or, + mask_or_assign, + mask_ternlog, + mask_ternlog_assign, + mask_xor, + mask_xor_assign, + masked_max_i32, + masked_min_i32, + masked_strided_group_sum, + masked_sum_i32, + ne_i32_to_mask, + ne_u32_to_mask, + ternary_match_strided_to_mask, + ternary_match_u32_to_mask, + ternary_match_u64_to_mask, }; // The popcount that closes the loop on the masks above: `mask_count` in ABI // terms. Already public at `ndarray::bitwise::popcount_batch_u64`; re-exported @@ -1433,6 +1510,107 @@ mod tests { ); } + /// The `I32x16` compare-to-bitmask and horizontal min/max methods at + /// LANE EXTREMES. On the AVX2 backend these are two-half intrinsic + /// realizations (PR #306, measured on the codegen oracle), and the two + /// ways such a body goes wrong — the halves concatenated in the wrong + /// order, or a reduction tree that drops a lane — are both invisible to a + /// splat-only test. So: every lane distinct, the signed extremes placed at + /// lane 0 / 7 / 8 / 15 (both edges of both halves), and the mask compared + /// bit-for-bit against the scalar definition, LSB-first. + #[test] + fn i32x16_compare_bitmasks_and_reductions_at_lane_extremes() { + use crate::simd::I32x16; + // Lanes 0/7/8/15 carry the extremes; the rest are distinct, mixed-sign. + let mut a = [0i32; 16]; + for (i, v) in a.iter_mut().enumerate() { + *v = (i as i32 - 8) * 1_000_003; + } + a[0] = i32::MIN; + a[7] = i32::MAX; + a[8] = -1; + a[15] = 0; + let mut b = a; + b.rotate_left(3); // same values, different lanes -> nontrivial gt pattern + let (va, vb) = (I32x16::from_array(a), I32x16::from_array(b)); + + let want_gt = (0..16).fold(0u16, |m, i| m | (((a[i] > b[i]) as u16) << i)); + let want_ge0 = (0..16).fold(0u16, |m, i| m | (((a[i] >= 0) as u16) << i)); + assert_eq!(va.gt_bitmask(vb), want_gt, "gt_bitmask lane order / sign"); + assert_eq!(vb.gt_bitmask(va), (0..16).fold(0u16, |m, i| m | (((b[i] > a[i]) as u16) << i))); + assert_eq!(va.cmpge_zero_mask(), want_ge0, "cmpge_zero_mask"); + // Anti-vacuity: the patterns must exercise both halves and both edges. + assert_ne!(want_gt & 0x00FF, 0); + assert_ne!(want_gt & 0xFF00, 0); + assert_ne!(want_gt, 0xFFFF); + assert_eq!(want_ge0 & 1, 0, "lane 0 is i32::MIN, must be clear"); + assert_ne!(want_ge0 & (1 << 15), 0, "lane 15 is 0, must be set"); + assert_ne!(want_ge0 & (1 << 7), 0, "lane 7 is i32::MAX, must be set"); + assert_eq!(want_ge0 & (1 << 8), 0, "lane 8 is -1, must be clear"); + + // Reductions: the extreme must be found wherever it sits, so walk it + // through every lane position (a tree that drops a lane fails here). + for pos in 0..16 { + let mut m = a; + m.swap(0, pos); // move i32::MIN to `pos` + let mut x = a; + x.swap(7, pos); // move i32::MAX to `pos` + assert_eq!(I32x16::from_array(m).reduce_min(), i32::MIN, "reduce_min with MIN at lane {pos}"); + assert_eq!(I32x16::from_array(x).reduce_max(), i32::MAX, "reduce_max with MAX at lane {pos}"); + } + assert_eq!(va.reduce_min(), *a.iter().min().unwrap()); + assert_eq!(va.reduce_max(), *a.iter().max().unwrap()); + } + + /// The same exhaustive sweep on the `U32x16` lane. On x86 this is a + /// DISTINCT body from the `U64x8` one (a second generated ladder and a + /// second two-input helper on the v3 arm; `_mm512_ternarylogic_epi32` vs + /// `_epi64` on v4), so the `U64x8` sweep above proves nothing about it — + /// before this test, x86 covered the `U32x16` body at two immediates. + #[test] + fn w1a9_u32x16_ternlog_matches_truth_table_reference_all_256_imms() { + let mut st = 0x5EED_0F32_5EED_0F32_u64; + let mut corpus: Vec<[u32; 16]> = Vec::with_capacity(24); + corpus.push([0xF0F0_F0F0; 16]); + corpus.push([0xCCCC_CCCC; 16]); + corpus.push([0xAAAA_AAAA; 16]); + for _ in 0..21 { + let mut a = [0u32; 16]; + for lane in a.iter_mut() { + *lane = splitmix64(&mut st) as u32; + } + corpus.push(a); + } + macro_rules! sweep { + ($($imm:literal),* $(,)?) => {$({ + for w in corpus.windows(3) { + let (a, b, c) = (w[0], w[1], w[2]); + let got = U32x16::from_array(a) + .ternlog::<$imm>(U32x16::from_array(b), U32x16::from_array(c)) + .to_array(); + for i in 0..16 { + let want = ref_ternlog_u64(a[i] as u64, b[i] as u64, c[i] as u64, $imm) as u32; + assert_eq!(got[i], want, "u32 ternlog imm={} lane={}", $imm, i); + } + } + })*}; + } + sweep!( + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, + 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, + 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, + 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, + 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, + 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, + 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, + 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, + 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, + 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 233, + 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 254, + 255 + ); + } + /// G3 — the named truth-table constants mean what their docs say, and each /// one is distinguishable from the others on real input (anti-vacuity: a /// table of aliases would pass a weaker test). diff --git a/src/simd_amx.rs b/src/simd_amx.rs index ffb3f6d1..d537e389 100644 --- a/src/simd_amx.rs +++ b/src/simd_amx.rs @@ -121,10 +121,43 @@ pub fn cpu_model() -> CpuModel { #[cfg(target_arch = "x86_64")] static AMX_AVAILABLE: std::sync::LazyLock = std::sync::LazyLock::new(detect_amx); +/// The tier-agnostic half of [`amx_available`], cached once: AMX-TILE on the +/// silicon, XSAVE + tile XSTATE enabled by the OS, and XTILEDATA permission +/// held by this process — everything a tile-STATE op (`ldtilecfg`, +/// `tilezero`, `tileloadd`, `tilestored`, `tilerelease`) needs, and nothing +/// about which compute tier exists. A host or hypervisor can expose TILE with +/// BF16 / FP16 / FP8 while masking INT8, so the compute tiers are gated +/// separately: INT8 by [`amx_available`], every other tier by this function +/// AND its `hpc::amx_ops::AmxFeatures` bit. +#[cfg(target_arch = "x86_64")] +static AMX_TILE_AVAILABLE: std::sync::LazyLock = std::sync::LazyLock::new(detect_amx_tile); + +/// AMX-TILE present, OS-enabled, and permitted for this process — the gate +/// for tile-state ops of ANY tier. See [`AMX_TILE_AVAILABLE`]; [`amx_available`] +/// is this plus the AMX-INT8 silicon bit. +/// +/// # Examples +/// +/// ``` +/// use ndarray::simd_amx::{amx_available, amx_tile_available}; +/// // INT8 availability implies tile availability, never the reverse. +/// if amx_available() { +/// assert!(amx_tile_available()); +/// } +/// ``` +#[cfg(target_arch = "x86_64")] +pub fn amx_tile_available() -> bool { + *AMX_TILE_AVAILABLE +} + /// Check if AMX is present, OS-enabled, AND this process holds XTILEDATA /// permission. Cached after the first call (see the `AMX_AVAILABLE` static). /// -/// Four gates, in order — any miss ⇒ `false`: +/// Four gates — any miss ⇒ `false`. Gates 1 (TILE only), 2, 3 and 4 run in +/// that order inside [`amx_tile_available`], which the non-INT8 compute tiers +/// gate on; the INT8 bit of gate 1 is consulted LAST, after the tile gate has +/// passed, so a TILE-only host takes the `arch_prctl` request but still +/// answers `false` here: /// 1. CPUID.07H.0H:EDX bits 24 (AMX-TILE) + 25 (AMX-INT8): silicon supports it. /// 2. CPUID.01H:ECX bit 27 (OSXSAVE): OS turned on XSAVE. /// 3. XGETBV(0) bits 17 (TILECFG) + 18 (TILEDATA): OS enabled tile XSTATE. @@ -140,14 +173,30 @@ pub fn amx_available() -> bool { *AMX_AVAILABLE } -/// The actual four-gate detection, run once behind the `AMX_AVAILABLE` static. +/// The actual four-gate detection, run once behind the `AMX_AVAILABLE` static: +/// the tier-agnostic tile gate, then the AMX-INT8 silicon bit. #[cfg(target_arch = "x86_64")] fn detect_amx() -> bool { - // Step 1: CPU supports AMX-TILE + AMX-INT8? + if !amx_tile_available() { + return false; + } + // CPUID.07H.0H:EDX bit 25 — the INT8 compute tier this crate's GEMM uses. + (core::arch::x86_64::__cpuid_count(7, 0).edx >> 25) & 1 == 1 +} + +/// Gates 1 (TILE only), 2, 3 and 4 of [`amx_available`], run once behind the +/// `AMX_TILE_AVAILABLE` static. +#[cfg(target_arch = "x86_64")] +fn detect_amx_tile() -> bool { + // Step 1: CPU supports AMX-TILE? (INT8 is checked by `detect_amx`, not here.) + // Leaf 7 is guarded by the max basic leaf, as `amx_ops::detect_amx_features` + // does: an out-of-range basic leaf may echo the highest leaf's data. + if core::arch::x86_64::__cpuid(0).eax < 7 { + return false; + } let cpuid = core::arch::x86_64::__cpuid_count(7, 0); let amx_tile = (cpuid.edx >> 24) & 1; - let amx_int8 = (cpuid.edx >> 25) & 1; - if amx_tile == 0 || amx_int8 == 0 { + if amx_tile == 0 { return false; } @@ -162,6 +211,9 @@ fn detect_amx() -> bool { // _xgetbv(0) reads the ACTUAL XCR0 register (what the OS set), // not the CPUID-reported capability. // Bit 17 = TILECFG, Bit 18 = TILEDATA. Both must be set. + // SAFETY: `_xgetbv` requires the `xsave` feature; XGETBV(0) is legal + // (never #UD) once CPUID.01H:ECX[27] (OSXSAVE) is set, which step 2 + // above has just verified — it is the only precondition the intrinsic has. let xcr0: u64 = unsafe { core::arch::x86_64::_xgetbv(0) }; let tilecfg = (xcr0 >> 17) & 1; let tiledata = (xcr0 >> 18) & 1; @@ -216,6 +268,12 @@ pub fn amx_available() -> bool { false } +/// Non-x86: no tiles. See the x86_64 [`amx_tile_available`]. +#[cfg(not(target_arch = "x86_64"))] +pub fn amx_tile_available() -> bool { + false +} + /// AMX capability report: detected CPU model + CPUID feature bits + the cached /// `amx_available()` verdict. If `model.has_amx()` is true but `available` is /// false, the gap is OS / hypervisor enablement (XCR0 / arch_prctl), not silicon. @@ -227,14 +285,22 @@ pub fn amx_report() -> String { let int8 = (cpuid.edx >> 25) & 1 == 1; let bf16 = (cpuid.edx >> 22) & 1 == 1; let model = cpu_model(); + let f = crate::hpc::amx_ops::amx_features(); format!( - "AMX [{} expects_amx={}]: TILE={} INT8={} BF16={} available={}", + "AMX [{} expects_amx={}]: TILE={} INT8={} BF16={} tile_available={} available={} | tiers: fp16={} complex={} fp8={} tf32={} avx512={} movrs={}", model.label(), model.has_amx(), tile, int8, bf16, - amx_available() + amx_tile_available(), + amx_available(), + f.fp16, + f.complex, + f.fp8, + f.tf32, + f.avx512, + f.movrs, ) } #[cfg(not(target_arch = "x86_64"))] diff --git a/src/simd_avx2.rs b/src/simd_avx2.rs index 9f372b6b..f0cf5d36 100644 --- a/src/simd_avx2.rs +++ b/src/simd_avx2.rs @@ -673,16 +673,26 @@ impl F32x16 { } F32Mask16(bits) } - /// Gather 16 f32 values from `base_ptr` using 16 i32 indices. + /// Gather 16 f32 values at `base_ptr.offset(indices[i])` — the same + /// signature and contract as the AVX-512 backend's `_mm512_i32gather_ps` + /// form: indices are SIGNED element offsets, so a negative index reads + /// an element before `base_ptr`. (A first cut cast each index to `usize` + /// and used `add`, which turned `-1` into a huge positive offset — + /// undefined behaviour on this backend for an index the other backends + /// accept; CodeRabbit on PR #306.) /// /// # Safety - /// Caller must ensure all indices are valid offsets into the memory at `base_ptr`. + /// For every `i in 0..16`, `base_ptr.offset(indices[i] as isize)` must + /// lie inside one allocation together with `base_ptr`, be 4-byte + /// aligned, and point at an initialised, readable `f32`. #[inline(always)] pub unsafe fn gather(indices: I32x16, base_ptr: *const f32) -> Self { let idx = indices.0; let mut o = [0.0f32; 16]; for i in 0..16 { - o[i] = *base_ptr.add(idx[i] as usize); + // SAFETY: the caller's contract above — each signed offset stays + // inside `base_ptr`'s allocation and points at a readable `f32`. + o[i] = unsafe { *base_ptr.offset(idx[i] as isize) }; } Self::from_array(o) } @@ -797,6 +807,27 @@ impl Default for F32x16 { #[derive(Copy, Clone, Debug)] pub struct F32Mask16(pub u16); impl F32Mask16 { + /// The mask as a packed 16-bit bitmask, LSB-first (bit `i` = lane `i`). + /// The one representation-independent reading of a compare result: every + /// backend stores its mask differently (`__mmask16`, `u16`, + /// `core::simd::Mask`), so callers combine and inspect masks through this + /// rather than the tuple field (the `aabb` broadphase read `.0` directly + /// and did not compile on the portable backend — fixed 2026-09-14). + /// + /// # Examples + /// Bit `i` is lane `i`: with lanes 0 and 15 below the threshold the + /// `simd_lt` mask reads `0b1000_0000_0000_0001`. + /// ```rust,ignore + /// let mut a = [10.0f32; 16]; + /// a[0] = -1.0; + /// a[15] = -1.0; + /// let m = F32x16::from_array(a).simd_lt(F32x16::splat(0.0)); + /// assert_eq!(m.to_bitmask(), 0b1000_0000_0000_0001); + /// ``` + #[inline(always)] + pub fn to_bitmask(self) -> u16 { + self.0 + } #[inline(always)] pub fn select(self, true_val: F32x16, false_val: F32x16) -> F32x16 { let t = true_val.to_array(); @@ -1542,21 +1573,79 @@ avx2_int_type!(U16x32, u16, 32, 0u16); avx2_int_type!(U32x16, u32, 16, 0u32); avx2_int_type!(U64x8, u64, 8, 0u64); -/// u64 ARX rotate — the BLAKE2b / argon2 lane. +/// u64 ARX rotate — the BLAKE2b / argon2 lane, and the mask family's +/// word rotate. /// -/// Scalar per-lane loops, and **measured not to vectorize**: the codegen -/// oracle tried three spellings (`u64::rotate_right`, an explicit shift-or -/// with a runtime amount, and the same with BLAKE2b's constants 32/24/16/63) -/// and every one came back 0 packed, one `rorq` per lane. LLVM declines the -/// 64-bit *operation*, not the rotate *idiom* — it folded two of the probes -/// into byte-identical code. +/// **Measured not to vectorize from scalar source**: the codegen oracle +/// (`.claude/knowledge/simd-codegen-oracle/`) tried three spellings +/// (`u64::rotate_right`, an explicit shift-or with a runtime amount, and the +/// same with BLAKE2b's constants 32/24/16/63) and every one came back +/// 0 packed, one `rorq`/`rolq` per lane — re-confirmed 2026-09-14 on the +/// shipped method itself (`rotate_left_lib_u64x8`: 0 packed / 8 `rolq`). +/// LLVM declines the 64-bit *operation*, not the rotate *idiom*. /// -/// So unlike every other lane-wise op in this crate, the scalar spec is NOT -/// the implementation here. The native `VPROLVQ`/`VPRORVQ` override lives on -/// `simd_avx512`'s `U64x8`, which is a real `__m512i`; these arms are the -/// correct-but-unvectorized fallback, and that is a known cost rather than an -/// oversight. See `.claude/knowledge/crypto-lane-status.md`. +/// So this is the one place in the AVX2 backend where the array polyfill's +/// lane loop is replaced by an intrinsic realization: AVX2 has no packed +/// 64-bit rotate, but it has uniform-count packed 64-bit shifts, so a rotate +/// is `vpsllq` + `vpsrlq` + `vpor` per 256-bit half — the same lowering LLVM +/// applies on its own to the u32 lane's `rotate_left(12)`. The native +/// `VPROLVQ`/`VPRORVQ` single-instruction form lives on `simd_avx512`'s +/// `U64x8`. Bit-exact with `u64::rotate_left` for every `n` (the count is +/// reduced mod 64 and the zero case returned early, so no shift ever reaches +/// 64). See `.claude/knowledge/crypto-lane-status.md`. impl U64x8 { + /// The two 256-bit halves of the 64-byte-aligned array, loaded once. + #[inline(always)] + fn avx2_halves(self) -> (__m256i, __m256i) { + // SAFETY: this file is the x86-64-v3 backend. `.cargo/config.toml` + // pins `-Ctarget-cpu=x86-64-v3` for the SUPPORTED x86_64 builds that + // select this arm, but that pin is not enforced by the arm's cfg — + // a build whose RUSTFLAGS replaced the config compiles this arm too + // and is "not a supported execution target for it (it would SIGILL)", + // as `simd.rs`'s arm note says. So the obligation is the CALLER's: + // AVX2 must be present at run time on any host this arm runs on + // (the same footing the native `U16x16` below already stands on). The + // memory half is proven here: the array is `#[repr(align(64))]` and + // 64 bytes long, so both 32-byte loads are in bounds (`loadu` needs + // no alignment regardless). + unsafe { + let p = self.0.as_ptr() as *const __m256i; + (_mm256_loadu_si256(p), _mm256_loadu_si256(p.add(1))) + } + } + + /// Store two 256-bit halves back into a fresh `[u64; 8]` (8-byte aligned + /// local; the `#[repr(align(64))]` lives on the wrapper it is moved into, + /// which is why the stores below are `storeu`, never `store`). + #[inline(always)] + fn from_avx2_halves(lo: __m256i, hi: __m256i) -> Self { + let mut o = [0u64; 8]; + // SAFETY: see `avx2_halves`; two 32-byte stores into a 64-byte array. + unsafe { + let p = o.as_mut_ptr() as *mut __m256i; + _mm256_storeu_si256(p, lo); + _mm256_storeu_si256(p.add(1), hi); + } + Self(o) + } + + /// `(x << n) | (x >> (64 - n))` per 64-bit lane on one 256-bit half, + /// with `1 <= n <= 63` guaranteed by the callers. `_mm256_sll_epi64` / + /// `_mm256_srl_epi64` take the count from the low 64 bits of an xmm + /// (uniform across lanes), which is exactly a runtime-variable rotate. + #[inline(always)] + fn rotl_half(v: __m256i, n: u32) -> __m256i { + debug_assert!((1..=63).contains(&n)); + // SAFETY: AVX2 (see `avx2_halves`). Shift counts are in `1..=63`, so + // neither packed shift is by 64 or more (which would zero the lane + // and break the rotate identity). + unsafe { + let l = _mm256_sll_epi64(v, _mm_cvtsi32_si128(n as i32)); + let r = _mm256_srl_epi64(v, _mm_cvtsi32_si128((64 - n) as i32)); + _mm256_or_si256(l, r) + } + } + /// Lane-wise left-rotate by `n` bits. `n` is taken mod 64. #[inline(always)] pub fn rotate_left(self, n: u32) -> Self { @@ -1564,12 +1653,8 @@ impl U64x8 { if n == 0 { return self; } - let a = self.to_array(); - let mut o = [0u64; 8]; - for i in 0..8 { - o[i] = a[i].rotate_left(n); - } - Self::from_array(o) + let (lo, hi) = self.avx2_halves(); + Self::from_avx2_halves(Self::rotl_half(lo, n), Self::rotl_half(hi, n)) } /// Lane-wise right-rotate by `n` bits — BLAKE2b's direction. @@ -1581,12 +1666,8 @@ impl U64x8 { if n == 0 { return self; } - let a = self.to_array(); - let mut o = [0u64; 8]; - for i in 0..8 { - o[i] = a[i].rotate_right(n); - } - Self::from_array(o) + let (lo, hi) = self.avx2_halves(); + Self::from_avx2_halves(Self::rotl_half(lo, 64 - n), Self::rotl_half(hi, 64 - n)) } } @@ -2257,13 +2338,45 @@ impl U16x32 { } impl I32x16 { + /// The two 256-bit halves of the 64-byte-aligned array, loaded once. + #[inline(always)] + fn avx2_halves(self) -> (__m256i, __m256i) { + // SAFETY: x86-64-v3 backend, AVX2 is a compile-time property (see + // `U64x8::avx2_halves`); the array is 64 bytes, both loads in bounds. + unsafe { + let p = self.0.as_ptr() as *const __m256i; + (_mm256_loadu_si256(p), _mm256_loadu_si256(p.add(1))) + } + } + + /// Horizontal signed minimum. `iter().min()` measured fully scalar on + /// the codegen oracle (17 `cmpl` on GPRs, 0 packed), so this is a + /// `vpminsd` tree: 16 → 8 → 4 → 2 → 1 lanes. Exact — min is order-free. #[inline(always)] pub fn reduce_min(self) -> i32 { - *self.0.iter().min().unwrap() + let (lo, hi) = self.avx2_halves(); + // SAFETY: AVX2 (see `avx2_halves`); pure register ops. + unsafe { + let m8 = _mm256_min_epi32(lo, hi); + let m4 = _mm_min_epi32(_mm256_castsi256_si128(m8), _mm256_extracti128_si256(m8, 1)); + let m2 = _mm_min_epi32(m4, _mm_shuffle_epi32(m4, 0b01_00_11_10)); + let m1 = _mm_min_epi32(m2, _mm_shuffle_epi32(m2, 0b00_00_00_01)); + _mm_cvtsi128_si32(m1) + } } + + /// Horizontal signed maximum — the `vpmaxsd` twin of [`Self::reduce_min`]. #[inline(always)] pub fn reduce_max(self) -> i32 { - *self.0.iter().max().unwrap() + let (lo, hi) = self.avx2_halves(); + // SAFETY: AVX2 (see `avx2_halves`); pure register ops. + unsafe { + let m8 = _mm256_max_epi32(lo, hi); + let m4 = _mm_max_epi32(_mm256_castsi256_si128(m8), _mm256_extracti128_si256(m8, 1)); + let m2 = _mm_max_epi32(m4, _mm_shuffle_epi32(m4, 0b01_00_11_10)); + let m1 = _mm_max_epi32(m2, _mm_shuffle_epi32(m2, 0b00_00_00_01)); + _mm_cvtsi128_si32(m1) + } } #[inline(always)] pub fn simd_min(self, other: Self) -> Self { @@ -2319,16 +2432,22 @@ impl I32x16 { o } - /// Mask: bit i set where lane i >= 0. + /// Mask: bit i set where lane i >= 0 (LSB-first, lane 0 = bit 0). + /// + /// `>= 0` is "sign bit clear", so this is the complement of the packed + /// sign-bit extraction `vmovmskps` performs on each 256-bit half. The + /// scalar-loop spelling measured MIXED on the codegen oracle (LLVM + /// vectorized lanes 1..=12 and peeled lanes 0 and 13..=15 into scalar + /// `shll`/`orl` bit assembly); this is the clean two-`vmovmskps` form. #[inline(always)] pub fn cmpge_zero_mask(self) -> u16 { - let mut mask = 0u16; - for i in 0..16 { - if self.0[i] >= 0 { - mask |= 1 << i; - } - } - mask + let (lo, hi) = self.avx2_halves(); + // SAFETY: AVX2 (see `avx2_halves`); the casts reinterpret bits only. + let neg = unsafe { + (_mm256_movemask_ps(_mm256_castsi256_ps(lo)) as u32) + | ((_mm256_movemask_ps(_mm256_castsi256_ps(hi)) as u32) << 8) + }; + !(neg as u16) } /// Lane-wise **signed** greater-than as a packed 16-bit bitmask. @@ -2343,20 +2462,25 @@ impl I32x16 { /// * `i32::MAX` as the threshold yields `0` — no `i32` exceeds it. /// * Comparison is signed, *not* bit-pattern: `-1 > 0` is `false`. /// - /// Plain index loop over the array polyfill — the codegen oracle - /// (`.claude/knowledge/simd-codegen-oracle/`) measured that LLVM lowers - /// compare-and-pack-to-bitmask shapes of exactly this form to packed - /// compares plus a `vmovmsk`-class extraction, so no `unsafe` and no - /// `core::arch` intrinsic override is earned here. + /// `vpcmpgtd` per 256-bit half, then `vmovmskps` on the all-ones/all-zeros + /// lanes — bit `i` of each 8-bit movemask is lane `i`'s sign bit, so the + /// two halves concatenate LSB-first with no reordering. + /// + /// The earlier index-loop spelling measured MIXED on the codegen oracle + /// (`gt_bitmask_i32x16`, 2026-09-14: 23 packed but lanes 0 and 13..=15 + /// peeled off into scalar compares and `shll`/`orl` assembly) — the doc + /// comment that stood here claimed a clean packed lowering, which the + /// measurement did not bear out. Hence the intrinsic realization. #[inline(always)] pub fn gt_bitmask(self, other: Self) -> u16 { - let mut mask = 0u16; - for i in 0..16 { - if self.0[i] > other.0[i] { - mask |= 1 << i; - } + let (a_lo, a_hi) = self.avx2_halves(); + let (b_lo, b_hi) = other.avx2_halves(); + // SAFETY: AVX2 (see `avx2_halves`); pure register ops. + unsafe { + let lo = _mm256_movemask_ps(_mm256_castsi256_ps(_mm256_cmpgt_epi32(a_lo, b_lo))) as u32; + let hi = _mm256_movemask_ps(_mm256_castsi256_ps(_mm256_cmpgt_epi32(a_hi, b_hi))) as u32; + (lo | (hi << 8)) as u16 } - mask } } impl Mul for I32x16 { @@ -3558,33 +3682,26 @@ impl U64x8 { #[inline(always)] pub fn ternlog(self, b: Self, c: Self) -> Self { const { assert!(IMM >= 0 && IMM <= 255, "ternlog IMM is an 8-bit truth table") } - let (a, z) = (self, Self::splat(0)); - let mut r = z; - if IMM & 0x01 != 0 { - r = r | !a & !b & !c; + // GENERATED lowering (tools/gen_ternlog_bodies.py): Shannon-expand on `c` + // into two 2-input tables; <= 8 ops for any table in this vocabulary + // (and-not is `x & !y`, two ops), folded at compile time. + let t0: u8 = ((IMM & 1) | ((IMM >> 1) & 2) | ((IMM >> 2) & 4) | ((IMM >> 3) & 8)) as u8; + let t1: u8 = (((IMM >> 1) & 1) | ((IMM >> 2) & 2) | ((IMM >> 3) & 4) | ((IMM >> 4) & 8)) as u8; + if t0 == t1 { + ternlog_two_input_u64x8(t0, self, b) + } else if t0 == 0 { + c & ternlog_two_input_u64x8(t1, self, b) + } else if t1 == 0 { + ternlog_two_input_u64x8(t0, self, b) & !c + } else if t1 == (t0 ^ 0xF) { + c ^ ternlog_two_input_u64x8(t0, self, b) + } else if t0 == 0xF { + !c | ternlog_two_input_u64x8(t1, self, b) + } else if t1 == 0xF { + c | ternlog_two_input_u64x8(t0, self, b) + } else { + (ternlog_two_input_u64x8(t0, self, b) & !c) | (ternlog_two_input_u64x8(t1, self, b) & c) } - if IMM & 0x02 != 0 { - r = r | !a & !b & c; - } - if IMM & 0x04 != 0 { - r = r | !a & b & !c; - } - if IMM & 0x08 != 0 { - r = r | !a & b & c; - } - if IMM & 0x10 != 0 { - r = r | a & !b & !c; - } - if IMM & 0x20 != 0 { - r = r | a & !b & c; - } - if IMM & 0x40 != 0 { - r = r | a & b & !c; - } - if IMM & 0x80 != 0 { - r = r | a & b & c; - } - r } } @@ -3617,32 +3734,79 @@ impl U32x16 { #[inline(always)] pub fn ternlog(self, b: Self, c: Self) -> Self { const { assert!(IMM >= 0 && IMM <= 255, "ternlog IMM is an 8-bit truth table") } - let (a, z) = (self, Self::splat(0)); - let mut r = z; - if IMM & 0x01 != 0 { - r = r | !a & !b & !c; - } - if IMM & 0x02 != 0 { - r = r | !a & !b & c; - } - if IMM & 0x04 != 0 { - r = r | !a & b & !c; - } - if IMM & 0x08 != 0 { - r = r | !a & b & c; - } - if IMM & 0x10 != 0 { - r = r | a & !b & !c; - } - if IMM & 0x20 != 0 { - r = r | a & !b & c; - } - if IMM & 0x40 != 0 { - r = r | a & b & !c; - } - if IMM & 0x80 != 0 { - r = r | a & b & c; - } - r - } -} + // GENERATED lowering (tools/gen_ternlog_bodies.py): Shannon-expand on `c` + // into two 2-input tables; <= 8 ops for any table in this vocabulary + // (and-not is `x & !y`, two ops), folded at compile time. + let t0: u8 = ((IMM & 1) | ((IMM >> 1) & 2) | ((IMM >> 2) & 4) | ((IMM >> 3) & 8)) as u8; + let t1: u8 = (((IMM >> 1) & 1) | ((IMM >> 2) & 2) | ((IMM >> 3) & 4) | ((IMM >> 4) & 8)) as u8; + if t0 == t1 { + ternlog_two_input_u32x16(t0, self, b) + } else if t0 == 0 { + c & ternlog_two_input_u32x16(t1, self, b) + } else if t1 == 0 { + ternlog_two_input_u32x16(t0, self, b) & !c + } else if t1 == (t0 ^ 0xF) { + c ^ ternlog_two_input_u32x16(t0, self, b) + } else if t0 == 0xF { + !c | ternlog_two_input_u32x16(t1, self, b) + } else if t1 == 0xF { + c | ternlog_two_input_u32x16(t0, self, b) + } else { + (ternlog_two_input_u32x16(t0, self, b) & !c) | (ternlog_two_input_u32x16(t1, self, b) & c) + } + } +} + +// GEN-TERNLOG-BEGIN (tools/gen_ternlog_bodies.py — regenerate, do not hand-edit) +/// GENERATED by `tools/gen_ternlog_bodies.py` — a 2-input Boolean function +/// by its 4-bit table (bit `k` = value at index `(a << 1) | b`), at most +/// two operations. `#[inline]` (not `always`): the 256-table test would +/// otherwise carry every arm's temporaries in one debug frame. +#[inline] +fn ternlog_two_input_u64x8(t: u8, a: U64x8, b: U64x8) -> U64x8 { + match t & 0xF { + 0x0 => U64x8::splat(0), + 0x1 => !(a | b), + 0x2 => !a & b, + 0x3 => !a, + 0x4 => a & !b, + 0x5 => !b, + 0x6 => a ^ b, + 0x7 => !(a & b), + 0x8 => a & b, + 0x9 => !(a ^ b), + 0xa => b, + 0xb => !a | b, + 0xc => a, + 0xd => a | !b, + 0xe => a | b, + _ => U64x8::splat(!0), + } +} + +/// GENERATED by `tools/gen_ternlog_bodies.py` — a 2-input Boolean function +/// by its 4-bit table (bit `k` = value at index `(a << 1) | b`), at most +/// two operations. `#[inline]` (not `always`): the 256-table test would +/// otherwise carry every arm's temporaries in one debug frame. +#[inline] +fn ternlog_two_input_u32x16(t: u8, a: U32x16, b: U32x16) -> U32x16 { + match t & 0xF { + 0x0 => U32x16::splat(0), + 0x1 => !(a | b), + 0x2 => !a & b, + 0x3 => !a, + 0x4 => a & !b, + 0x5 => !b, + 0x6 => a ^ b, + 0x7 => !(a & b), + 0x8 => a & b, + 0x9 => !(a ^ b), + 0xa => b, + 0xb => !a | b, + 0xc => a, + 0xd => a | !b, + 0xe => a | b, + _ => U32x16::splat(!0), + } +} +// GEN-TERNLOG-END diff --git a/src/simd_avx512.rs b/src/simd_avx512.rs index 71f92cc9..9cc0ef69 100644 --- a/src/simd_avx512.rs +++ b/src/simd_avx512.rs @@ -287,6 +287,29 @@ impl PartialEq for F32x16 { pub struct F32Mask16(pub __mmask16); impl F32Mask16 { + /// The mask as a packed 16-bit bitmask, LSB-first (bit `i` = lane `i`). + /// The one representation-independent reading of a compare result: every + /// backend stores its mask differently (`__mmask16`, `u16`, + /// `core::simd::Mask`), so callers combine and inspect masks through this + /// rather than the tuple field (the `aabb` broadphase read `.0` directly + /// and did not compile on the portable backend — fixed 2026-09-14). + /// + /// # Examples + /// Bit `i` is lane `i`: with lanes 0 and 15 below the threshold the + /// `simd_lt` mask reads `0b1000_0000_0000_0001`. `ignore`d rather than + /// `no_run` because this type exists only under a compile-time `avx512f` + /// (v4 / native builds); a v3 doctest build cannot see it. + /// ```rust,ignore + /// let mut a = [10.0f32; 16]; + /// a[0] = -1.0; + /// a[15] = -1.0; + /// let m = F32x16::from_array(a).simd_lt(F32x16::splat(0.0)); + /// assert_eq!(m.to_bitmask(), 0b1000_0000_0000_0001); + /// ``` + #[inline(always)] + pub fn to_bitmask(self) -> u16 { + self.0 + } /// Select: for each lane, if mask bit is 1 → true_val, else false_val. #[inline(always)] pub fn select(self, true_val: F32x16, false_val: F32x16) -> F32x16 { diff --git a/src/simd_int_ops.rs b/src/simd_int_ops.rs index 3921f241..390c71d5 100644 --- a/src/simd_int_ops.rs +++ b/src/simd_int_ops.rs @@ -13,6 +13,15 @@ //! deliberately wider than the lane element type — `127 × 127 × 64 ≈ 1 M` //! fits in i32 but not in i8/i16 reductions. +// The mask family lived here until PR #306 moved it to `simd_masking_ops`. +// `simd_int_ops` is a `pub mod`, so `ndarray::simd_int_ops::` was a +// public path; these re-exports keep every such path compiling. The canonical +// path is `ndarray::simd::` (the facade) — new code uses that. +pub use crate::simd_masking_ops::{ + eq_u32_strided_to_mask, eq_u32_to_mask, gt_i32_to_mask, mask_and, mask_and_assign, mask_andnot, mask_andnot_assign, + mask_or, mask_or_assign, mask_ternlog, mask_ternlog_assign, masked_strided_group_sum, masked_sum_i32, +}; + // ──────────────────────────────────────────────────────────────────────── // add_i8 / sub_i8 — element-wise mutate-in-place // ──────────────────────────────────────────────────────────────────────── @@ -462,801 +471,6 @@ pub fn max_i8(s: &[i8]) -> i8 { m } -// ──────────────────────────────────────────────────────────────────────── -// Packed-bitmask predicates + mask algebra (the columnar-selection lane) -// ──────────────────────────────────────────────────────────────────────── -// -// These seven primitives are the vector half of a columnar filter: turn a -// lane of values into a packed bit-per-row mask, compose masks with boolean -// algebra, and reduce a value lane under a mask. They are the substrate the -// `lance-graph-java` ABI membrane rides (`lgj_op_eq_u32`, `lgj_op_gt_i32`, -// `lgj_mask_and`, `lgj_mask_or`, `lgj_plan_eval`, `lgj_reduce_sum_i32`), and -// the reason that membrane needs no SIMD of its own — a consumer crate that -// wrote its own compare-and-pack loop would be an `ndarray::simd` bypass. -// -// ## Bit order (NORMATIVE — every function below obeys it) -// -// Element index `i` lives at **bit `i % 64` of word `i / 64`**; LSB-first -// within each word, so element 0 is bit 0 of `out_words[0]` and element 64 is -// bit 0 of `out_words[1]`. This matches the `MASK_WORD` lane definition on -// the ABI side ("a `u64` of 64 packed row bits, LSB = lowest row index") and -// the lane-level `u16` convention already established by -// `I32x16::cmpge_zero_mask`. -// -// **Trailing bits beyond `values.len()` in the final word are always written -// as 0**, as are any surplus words in a longer-than-necessary `out_words`. -// This is load-bearing: those bits feed straight into `popcount_batch_u64`, -// so a stale high bit would silently inflate a count. Every writer below -// zeroes the whole destination first and then only ever sets bits for -// in-range elements, which makes the guarantee structural rather than a -// tail-handling special case that could be forgotten. -// -// ## Why free functions here, not methods on a wrapper -// -// The W1a consumer contract's "struct method, not free function" litmus -// governs **lane-level** primitives, where a free function fragments the -// typed-wrapper surface. These are **slice-level**, the same tier as -// `add_i8` / `dot_i8` / `min_i8` above, and they are built *on* lane methods -// (`U32x16::eq_bitmask`, `I32x16::gt_bitmask`) that do live on the wrappers. - -/// Number of packed mask words needed to cover `n` elements. -#[inline(always)] -fn mask_words_for(n: usize) -> usize { - n.div_ceil(64) -} - -/// Load 16 `u32` lanes from the front of `src`. -/// -/// Uses `from_array` rather than `from_slice` deliberately: `from_slice` is -/// not present on every backend's `U32x16` (the NEON and wasm `[U32x4; 4]` -/// fan-outs expose `from_array` only), and going through the array keeps this -/// helper free of any `cfg(target_arch)` selection. The 64-byte copy is -/// elided into a single vector load by LLVM. -#[inline(always)] -fn load_u32x16(src: &[u32]) -> crate::simd::U32x16 { - let mut a = [0u32; 16]; - a.copy_from_slice(&src[..16]); - crate::simd::U32x16::from_array(a) -} - -/// Load 16 `i32` lanes from the front of `src`. See [`load_u32x16`]. -#[inline(always)] -fn load_i32x16(src: &[i32]) -> crate::simd::I32x16 { - let mut a = [0i32; 16]; - a.copy_from_slice(&src[..16]); - crate::simd::I32x16::from_array(a) -} - -/// Packs `values[i] == needle` into `out_words`, one bit per element, -/// LSB-first within each `u64` word (bit `k` of word `w` corresponds to -/// element `w * 64 + k`). -/// -/// `out_words` is **fully overwritten**, not OR-ed into. Trailing bits in the -/// final word beyond `values.len()`, and any surplus words past -/// `ceil(len / 64)`, are written as `0`. -/// -/// Equality is exact bitwise comparison over the full `u32` range — `0` and -/// `u32::MAX` are ordinary needles, and there is no saturation, wrapping, or -/// signedness question to resolve. An empty `values` writes only zeros. -/// -/// Runs 16 lanes at a time through [`crate::simd::U32x16::eq_bitmask`] with a -/// scalar tail for the final partial group; the scalar tail is bit-identical -/// to the vector path by construction (same comparison, same bit index). -/// -/// # Panics -/// -/// Panics if `out_words.len() < values.len().div_ceil(64)`. -/// -/// # Examples -/// -/// ``` -/// use ndarray::simd::eq_u32_to_mask; -/// -/// let values = [7u32, 1, 7, 2]; -/// let mut words = [0u64; 1]; -/// eq_u32_to_mask(&values, 7, &mut words); -/// // elements 0 and 2 match → bits 0 and 2 → 0b0101 -/// assert_eq!(words[0], 0b0101); -/// ``` -#[inline] -pub fn eq_u32_to_mask(values: &[u32], needle: u32, out_words: &mut [u64]) { - let n = values.len(); - let words = mask_words_for(n); - assert!(out_words.len() >= words, "eq_u32_to_mask: out_words.len()={} < required {}", out_words.len(), words); - - // Zero first: makes the "trailing bits are 0" guarantee structural. - for w in out_words.iter_mut() { - *w = 0; - } - - let needle_v = crate::simd::U32x16::splat(needle); - let groups = n / 16; - for g in 0..groups { - let bits = load_u32x16(&values[g * 16..]).eq_bitmask(needle_v); - out_words[g / 4] |= (bits as u64) << ((g % 4) * 16); - } - for i in (groups * 16)..n { - if values[i] == needle { - out_words[i / 64] |= 1u64 << (i % 64); - } - } -} - -/// Packs `read_le_u32(bytes, first_offset + i * stride_bytes) == needle` into -/// `out_words`, one bit per element, LSB-first within each `u64` word — the -/// **strided** sibling of [`eq_u32_to_mask`], for scanning one `u32` field of -/// an AoS/facet row layout (e.g. a 4-byte classid at a fixed offset inside a -/// 512-byte row) without gathering the column into a contiguous copy first. -/// -/// Element `i` is the little-endian `u32` at byte offset -/// `first_offset + i * stride_bytes`. `stride_bytes == 4` reads a contiguous -/// `u32` column and takes a dedicated contiguous path (one 64-byte window per -/// 16 elements — the same load shape as [`eq_u32_to_mask`]); `stride_bytes -/// == 0` re-reads the same field `count` times, which is legal and produces -/// an all-ones or all-zeros mask. -/// -/// `out_words` is **fully overwritten**, not OR-ed into; trailing bits and -/// surplus words are written `0`, exactly as in [`eq_u32_to_mask`]. -/// -/// The field loads are scalar by construction — at row strides ≥ one cache -/// line each element lives on its own line, so the walk is memory-bound and -/// a hardware gather buys nothing; SIMD earns its keep in the 16-wide -/// compare ([`crate::simd::U32x16::eq_bitmask`]) exactly as the contiguous -/// primitive does. Loads are `u32::from_le_bytes` over byte slices, so no -/// alignment is required of `bytes`. -/// -/// # Panics -/// -/// Panics if `out_words.len() < count.div_ceil(64)`, or if any element's four -/// bytes would fall outside `bytes` (checked up front, including overflow of -/// the offset arithmetic — the loop never reads out of bounds). -/// -/// # Examples -/// -/// ``` -/// use ndarray::simd::eq_u32_strided_to_mask; -/// -/// // Three 16-byte "facets"; the classid is the leading u32 of each. -/// let mut rows = vec![0u8; 48]; -/// rows[0..4].copy_from_slice(&7u32.to_le_bytes()); -/// rows[16..20].copy_from_slice(&9u32.to_le_bytes()); -/// rows[32..36].copy_from_slice(&7u32.to_le_bytes()); -/// let mut words = [0u64; 1]; -/// eq_u32_strided_to_mask(&rows, 0, 16, 3, 7, &mut words); -/// assert_eq!(words[0], 0b101); -/// ``` -#[inline] -pub fn eq_u32_strided_to_mask( - bytes: &[u8], first_offset: usize, stride_bytes: usize, count: usize, needle: u32, out_words: &mut [u64], -) { - let words = mask_words_for(count); - assert!( - out_words.len() >= words, - "eq_u32_strided_to_mask: out_words.len()={} < required {}", - out_words.len(), - words - ); - if count > 0 { - // Bounds of the LAST element, computed with overflow checks so a - // pathological stride cannot wrap around into a bogus in-bounds read. - let last_start = (count - 1) - .checked_mul(stride_bytes) - .and_then(|o| o.checked_add(first_offset)) - .expect("eq_u32_strided_to_mask: offset arithmetic overflow"); - let last_end = last_start - .checked_add(4) - .expect("eq_u32_strided_to_mask: offset arithmetic overflow"); - assert!( - last_end <= bytes.len(), - "eq_u32_strided_to_mask: element {} at byte {}..{} is out of bounds (len {})", - count - 1, - last_start, - last_end, - bytes.len() - ); - } - - for w in out_words.iter_mut() { - *w = 0; - } - - #[inline(always)] - fn read_le_u32(bytes: &[u8], off: usize) -> u32 { - u32::from_le_bytes([bytes[off], bytes[off + 1], bytes[off + 2], bytes[off + 3]]) - } - - let needle_v = crate::simd::U32x16::splat(needle); - let groups = count / 16; - if stride_bytes == 4 { - // Contiguous lane (a facet-major column): 16 elements are ONE 64-byte - // window. Alias it as a fixed-size array so the compiler emits a single - // vector load instead of the general path's 16 bounds-checked scalar - // reads gathered into a temporary — a cast, not a copy. Bounds were - // proven above for the last element, so `try_into` cannot fail here. - for g in 0..groups { - let base = first_offset + g * 64; - let window: &[u8; 64] = bytes[base..base + 64] - .try_into() - .expect("64-byte window proven in bounds"); - let lanes: [u32; 16] = core::array::from_fn(|k| { - u32::from_le_bytes([window[4 * k], window[4 * k + 1], window[4 * k + 2], window[4 * k + 3]]) - }); - let bits = crate::simd::U32x16::from_array(lanes).eq_bitmask(needle_v); - out_words[g / 4] |= (bits as u64) << ((g % 4) * 16); - } - } else { - for g in 0..groups { - let base = first_offset + g * 16 * stride_bytes; - let lanes: [u32; 16] = core::array::from_fn(|k| read_le_u32(bytes, base + k * stride_bytes)); - let bits = crate::simd::U32x16::from_array(lanes).eq_bitmask(needle_v); - out_words[g / 4] |= (bits as u64) << ((g % 4) * 16); - } - } - for i in (groups * 16)..count { - if read_le_u32(bytes, first_offset + i * stride_bytes) == needle { - out_words[i / 64] |= 1u64 << (i % 64); - } - } -} - -/// Packs `values[i] > threshold` (**signed** comparison) into `out_words`, -/// one bit per element, LSB-first within each `u64` word (bit `k` of word `w` -/// corresponds to element `w * 64 + k`). -/// -/// `out_words` is **fully overwritten**, not OR-ed into. Trailing bits in the -/// final word beyond `values.len()`, and any surplus words past -/// `ceil(len / 64)`, are written as `0`. -/// -/// Comparison is two's-complement signed and strict (`>`, never `>=`); it is -/// exact with no saturation or wrapping: -/// * `threshold == i32::MIN` sets every lane except those equal to `i32::MIN`. -/// * `threshold == i32::MAX` sets nothing — no `i32` exceeds it. -/// * Negative values compare as signed, *not* as bit patterns: `-1 > 0` is -/// `false` even though the same bits compare greater unsigned. -/// -/// An empty `values` writes only zeros. -/// -/// Runs 16 lanes at a time through [`crate::simd::I32x16::gt_bitmask`] with a -/// scalar tail for the final partial group. -/// -/// # Panics -/// -/// Panics if `out_words.len() < values.len().div_ceil(64)`. -/// -/// # Examples -/// -/// ``` -/// use ndarray::simd::gt_i32_to_mask; -/// -/// let values = [5i32, -5, 0, i32::MAX]; -/// let mut words = [0u64; 1]; -/// gt_i32_to_mask(&values, 0, &mut words); -/// // elements 0 and 3 exceed 0 → bits 0 and 3 → 0b1001 -/// assert_eq!(words[0], 0b1001); -/// ``` -#[inline] -pub fn gt_i32_to_mask(values: &[i32], threshold: i32, out_words: &mut [u64]) { - let n = values.len(); - let words = mask_words_for(n); - assert!(out_words.len() >= words, "gt_i32_to_mask: out_words.len()={} < required {}", out_words.len(), words); - - for w in out_words.iter_mut() { - *w = 0; - } - - let threshold_v = crate::simd::I32x16::splat(threshold); - let groups = n / 16; - for g in 0..groups { - let bits = load_i32x16(&values[g * 16..]).gt_bitmask(threshold_v); - out_words[g / 4] |= (bits as u64) << ((g % 4) * 16); - } - for i in (groups * 16)..n { - if values[i] > threshold { - out_words[i / 64] |= 1u64 << (i % 64); - } - } -} - -/// `dst = a & b`, elementwise over `u64` mask words. -/// -/// Pure bitwise AND — no element-count awareness, so the caller's bit-order -/// convention (element `i` at bit `i % 64` of word `i / 64`) is preserved -/// automatically, including the trailing-zero guarantee: zero AND anything is -/// zero, so a conforming pair of inputs yields a conforming output. -/// -/// `dst` must **not** overlap `a` or `b`; use [`mask_and_assign`] for the -/// in-place case (Rust's borrow rules already prevent the overlap in safe -/// code, so this is a note about which function to reach for, not a hazard). -/// -/// # Panics -/// -/// Panics unless `a.len() == b.len() == dst.len()`. -#[inline] -pub fn mask_and(a: &[u64], b: &[u64], dst: &mut [u64]) { - assert_eq!(a.len(), b.len(), "mask_and: a/b length mismatch"); - assert_eq!(a.len(), dst.len(), "mask_and: a/dst length mismatch"); - let n = a.len(); - - const L: usize = crate::simd::U64x8::LANES; - let groups = n / L; - for g in 0..groups { - let off = g * L; - let va = crate::simd::U64x8::from_slice(&a[off..]); - let vb = crate::simd::U64x8::from_slice(&b[off..]); - (va & vb).copy_to_slice(&mut dst[off..]); - } - for i in (groups * L)..n { - dst[i] = a[i] & b[i]; - } -} - -/// `dst = a | b`, elementwise over `u64` mask words. -/// -/// Pure bitwise OR. Note the trailing-zero asymmetry versus [`mask_and`]: OR -/// preserves the guarantee only if **both** inputs already conform, because a -/// stray high bit in either operand survives. Every mask this module produces -/// conforms, so composing them is safe; a hand-built mask word is the caller's -/// responsibility. -/// -/// `dst` must not overlap `a` or `b`; use [`mask_or_assign`] in-place. -/// -/// # Panics -/// -/// Panics unless `a.len() == b.len() == dst.len()`. -#[inline] -pub fn mask_or(a: &[u64], b: &[u64], dst: &mut [u64]) { - assert_eq!(a.len(), b.len(), "mask_or: a/b length mismatch"); - assert_eq!(a.len(), dst.len(), "mask_or: a/dst length mismatch"); - let n = a.len(); - - const L: usize = crate::simd::U64x8::LANES; - let groups = n / L; - for g in 0..groups { - let off = g * L; - let va = crate::simd::U64x8::from_slice(&a[off..]); - let vb = crate::simd::U64x8::from_slice(&b[off..]); - (va | vb).copy_to_slice(&mut dst[off..]); - } - for i in (groups * L)..n { - dst[i] = a[i] | b[i]; - } -} - -/// `dst &= src`, elementwise over `u64` mask words. -/// -/// The in-place form of [`mask_and`] — this is what a fused predicate plan -/// uses to narrow an accumulator, and what an ABI-level `mask_and(a, b, dst)` -/// with `dst` aliasing an operand must route to. -/// -/// # Panics -/// -/// Panics if `dst.len() != src.len()`. -#[inline] -pub fn mask_and_assign(dst: &mut [u64], src: &[u64]) { - assert_eq!(dst.len(), src.len(), "mask_and_assign: length mismatch"); - let n = dst.len(); - - const L: usize = crate::simd::U64x8::LANES; - let groups = n / L; - for g in 0..groups { - let off = g * L; - let vd = crate::simd::U64x8::from_slice(&dst[off..]); - let vs = crate::simd::U64x8::from_slice(&src[off..]); - (vd & vs).copy_to_slice(&mut dst[off..]); - } - for i in (groups * L)..n { - dst[i] &= src[i]; - } -} - -/// `dst |= src`, elementwise over `u64` mask words. -/// -/// The in-place form of [`mask_or`]. Same trailing-zero caveat as `mask_or`: -/// OR only preserves the convention if `src` conforms to it. -/// -/// # Panics -/// -/// Panics if `dst.len() != src.len()`. -#[inline] -pub fn mask_or_assign(dst: &mut [u64], src: &[u64]) { - assert_eq!(dst.len(), src.len(), "mask_or_assign: length mismatch"); - let n = dst.len(); - - const L: usize = crate::simd::U64x8::LANES; - let groups = n / L; - for g in 0..groups { - let off = g * L; - let vd = crate::simd::U64x8::from_slice(&dst[off..]); - let vs = crate::simd::U64x8::from_slice(&src[off..]); - (vd | vs).copy_to_slice(&mut dst[off..]); - } - for i in (groups * L)..n { - dst[i] |= src[i]; - } -} - -/// `dst = a & !b`, elementwise over `u64` mask words — "a minus b" as a -/// bitmask set difference (every bit set in `a` but not in `b`). -/// -/// # Tail-bit semantics -/// -/// `!b` sets every bit of `b`'s tail — the padding bits past whatever -/// logical row count `b` represents — because bitwise NOT has no notion of -/// "past the end" and will happily flip a conforming (zero) tail to all -/// ones. That looks like the same hazard [`mask_or`] warns about, but the -/// AND with `a` recovers it: `a & !b` is a bitwise subset of `a` (every bit -/// set in the result is also set in `a`), so **`dst`'s tail is zero -/// whenever `a`'s tail is zero, regardless of what `!b`'s tail does.** This -/// is the same pre-conforming-inputs contract `mask_or` documents — a -/// caller holding a possibly-non-conforming `a` must clear `a`'s tail -/// itself (the lgj-abi kernel does, against its own known `n_rows`); a -/// conforming `a` composes safely against any `b`, tail included. -/// -/// `dst` must not overlap `a` or `b`; use [`mask_andnot_assign`] for the -/// in-place case (Rust's borrow rules already prevent the overlap in safe -/// code, so this is a note about which function to reach for, not a -/// hazard). -/// -/// # Panics -/// -/// Panics unless `a.len() == b.len() == dst.len()`. -#[inline] -pub fn mask_andnot(a: &[u64], b: &[u64], dst: &mut [u64]) { - assert_eq!(a.len(), b.len(), "mask_andnot: a/b length mismatch"); - assert_eq!(a.len(), dst.len(), "mask_andnot: a/dst length mismatch"); - let n = a.len(); - - const L: usize = crate::simd::U64x8::LANES; - let groups = n / L; - for g in 0..groups { - let off = g * L; - let va = crate::simd::U64x8::from_slice(&a[off..]); - let vb = crate::simd::U64x8::from_slice(&b[off..]); - (va & !vb).copy_to_slice(&mut dst[off..]); - } - for i in (groups * L)..n { - dst[i] = a[i] & !b[i]; - } -} - -/// `a &= !b`, elementwise over `u64` mask words. -/// -/// The in-place form of [`mask_andnot`] — same tail-bit contract: the -/// result is a bitwise subset of the (pre-update) `a`, so `a`'s tail stays -/// zero whenever it started zero, regardless of what `b`'s tail holds. -/// -/// # Panics -/// -/// Panics if `a.len() != b.len()`. -#[inline] -pub fn mask_andnot_assign(a: &mut [u64], b: &[u64]) { - assert_eq!(a.len(), b.len(), "mask_andnot_assign: length mismatch"); - let n = a.len(); - - const L: usize = crate::simd::U64x8::LANES; - let groups = n / L; - for g in 0..groups { - let off = g * L; - let va = crate::simd::U64x8::from_slice(&a[off..]); - let vb = crate::simd::U64x8::from_slice(&b[off..]); - (va & !vb).copy_to_slice(&mut a[off..]); - } - for i in (groups * L)..n { - a[i] &= !b[i]; - } -} - -/// `dst = ternlog::(a, b, c)`, elementwise over `u64` mask words — any -/// 3-input Boolean function of three masks in one pass. -/// -/// `IMM` is the 8-bit truth table in Intel's VPTERNLOG convention (index -/// `(a<<2)|(b<<1)|c`, result bit `(IMM >> index) & 1`); the named tables in -/// [`crate::simd::ternlog`] (`AND3`, `OR3`, `MAJ3`, `AND2_ANDNOT`, …) are -/// the sanctioned spellings. This is the mask-op family's general member: -/// [`mask_and`] is `mask_ternlog::<{ ternlog::AND2 }>` with `c` ignored, -/// [`mask_andnot`] is `AND2_ANDNOT` with `c` ignored, and the composed -/// `a & b & c` that a consumer would otherwise spell as two `mask_and_assign` -/// passes through a scratch buffer is ONE `AND3` pass here — one -/// `VPTERNLOGQ` per 512 bits on AVX-512, the polyfill elsewhere. -/// -/// # Tail-bit semantics -/// -/// Whether `dst`'s tail conforms depends on the truth table, not on the -/// inputs alone: the tail of every conforming input is zero, so `dst`'s tail -/// is `IMM & 1` replicated — **zero iff `IMM` is even** (index 0 = all-zero -/// inputs maps to 0). Every named table in [`crate::simd::ternlog`] is even. -/// An odd `IMM` (one whose function is true of `(0,0,0)`) sets every tail bit -/// and the caller must clear the tail against its own known row count, exactly -/// as [`mask_or`] documents for a non-conforming operand. For the -/// subset-shaped tables (`AND3`, `AND2_ANDNOT`, `AND_ANDNOT2`, `AND2`) the -/// stronger [`mask_andnot`] guarantee also holds: the result is a bitwise -/// subset of `a`, so `dst`'s tail is zero whenever `a`'s is, regardless of -/// `b` and `c`. -/// -/// `dst` must not overlap `a`, `b` or `c`; use [`mask_ternlog_assign`] for -/// the in-place case. -/// -/// # Panics -/// -/// Panics unless `a.len() == b.len() == c.len() == dst.len()`. -#[inline] -pub fn mask_ternlog(a: &[u64], b: &[u64], c: &[u64], dst: &mut [u64]) { - assert_eq!(a.len(), b.len(), "mask_ternlog: a/b length mismatch"); - assert_eq!(a.len(), c.len(), "mask_ternlog: a/c length mismatch"); - assert_eq!(a.len(), dst.len(), "mask_ternlog: a/dst length mismatch"); - let n = a.len(); - - const L: usize = crate::simd::U64x8::LANES; - let groups = n / L; - for g in 0..groups { - let off = g * L; - let va = crate::simd::U64x8::from_slice(&a[off..]); - let vb = crate::simd::U64x8::from_slice(&b[off..]); - let vc = crate::simd::U64x8::from_slice(&c[off..]); - va.ternlog::(vb, vc).copy_to_slice(&mut dst[off..]); - } - for i in (groups * L)..n { - dst[i] = ternlog_word::(a[i], b[i], c[i]); - } -} - -/// `a = ternlog::(a, b, c)`, elementwise over `u64` mask words. -/// -/// The in-place form of [`mask_ternlog`] — `a` is the first truth-table -/// operand AND the destination, which is the shape a fused predicate plan -/// wants when narrowing an accumulator against two more masks in one pass -/// (`selected = selected & src & gate` as `AND3`). Same tail contract as -/// [`mask_ternlog`]. -/// -/// # Panics -/// -/// Panics unless `a.len() == b.len() == c.len()`. -#[inline] -pub fn mask_ternlog_assign(a: &mut [u64], b: &[u64], c: &[u64]) { - assert_eq!(a.len(), b.len(), "mask_ternlog_assign: a/b length mismatch"); - assert_eq!(a.len(), c.len(), "mask_ternlog_assign: a/c length mismatch"); - let n = a.len(); - - const L: usize = crate::simd::U64x8::LANES; - let groups = n / L; - for g in 0..groups { - let off = g * L; - let va = crate::simd::U64x8::from_slice(&a[off..]); - let vb = crate::simd::U64x8::from_slice(&b[off..]); - let vc = crate::simd::U64x8::from_slice(&c[off..]); - va.ternlog::(vb, vc).copy_to_slice(&mut a[off..]); - } - for i in (groups * L)..n { - a[i] = ternlog_word::(a[i], b[i], c[i]); - } -} - -/// One `u64` of the truth-table function — the scalar tail of the two -/// `mask_ternlog` forms, and the independent reference their parity test is -/// checked against. Bit-serial over the eight table rows, so it cannot share -/// a bug with any backend's lane implementation. -#[inline(always)] -fn ternlog_word(a: u64, b: u64, c: u64) -> u64 { - const { assert!(IMM >= 0 && IMM <= 255, "ternlog IMM is an 8-bit truth table") } - let mut r = 0u64; - if IMM & 0x01 != 0 { - r |= !a & !b & !c; - } - if IMM & 0x02 != 0 { - r |= !a & !b & c; - } - if IMM & 0x04 != 0 { - r |= !a & b & !c; - } - if IMM & 0x08 != 0 { - r |= !a & b & c; - } - if IMM & 0x10 != 0 { - r |= a & !b & !c; - } - if IMM & 0x20 != 0 { - r |= a & !b & c; - } - if IMM & 0x40 != 0 { - r |= a & b & !c; - } - if IMM & 0x80 != 0 { - r |= a & b & c; - } - r -} - -/// Sum of `values[i]` where mask bit `i` is set, widened to `i64`. -/// -/// Bit order is the module convention: element `i` is bit `i % 64` of -/// `mask_words[i / 64]`. -/// -/// ## Overflow behaviour (precise) -/// -/// Each element is widened to `i64` **before** accumulation, so no -/// intermediate can overflow at any realistic length: the worst case is -/// `n × |i32::MIN|`, which stays inside `i64` for every `n < 2^32` — i.e. for -/// every slice that can exist in a 64-bit address space at 4 bytes per -/// element. The accumulation is nevertheless written as `wrapping_add` so -/// that the theoretical `n ≥ 2^32` case has defined behaviour (two's-complement -/// wrap) rather than a debug-only panic that a release build would silently -/// disagree with. An empty mask, or a mask with no bits set, returns `0`. -/// -/// **Mask bits at or beyond `values.len()` are ignored**, not summed and not -/// an error: the final word is masked down to the valid element count before -/// its bits are walked. This makes the function total for any conforming or -/// over-long mask, and means a caller cannot read past the value lane by -/// handing over a dirty tail. -/// -/// ## Why this one is not a 16-lane reduce -/// -/// The obvious vector shape — load `I32x16`, zero the unselected lanes, -/// `reduce_sum()` — is **wrong**, and quietly so: `reduce_sum` on `I32x16` -/// accumulates in `i32`, and 16 lanes near `i32::MAX` overflow it while the -/// widened contract promises they cannot. Preserving the `i64` guarantee is -/// worth more than the lanes here, so the body walks set bits with -/// `u64::trailing_zeros` (one `TZCNT`/`RBIT+CLZ` per selected element, and -/// entire zero words skipped in one test). Cost is proportional to the -/// popcount, not the row count, which is the right shape for a selective -/// filter anyway. -/// -/// # Panics -/// -/// Panics if `mask_words.len() < values.len().div_ceil(64)`. -/// -/// # Examples -/// -/// ``` -/// use ndarray::simd::masked_sum_i32; -/// -/// let values = [10i32, 20, 30, 40]; -/// // bits 0 and 2 set → 10 + 30 -/// assert_eq!(masked_sum_i32(&values, &[0b0101]), 40); -/// ``` -#[inline] -pub fn masked_sum_i32(values: &[i32], mask_words: &[u64]) -> i64 { - let n = values.len(); - let words = mask_words_for(n); - assert!( - mask_words.len() >= words, - "masked_sum_i32: mask_words.len()={} < required {}", - mask_words.len(), - words - ); - - let mut acc: i64 = 0; - for (w, &word) in mask_words.iter().take(words).enumerate() { - let base = w * 64; - let mut bits = word; - // Clamp the final partial word to the valid element count so a dirty - // tail can never index past `values`. - let valid = n - base; - if valid < 64 { - bits &= (1u64 << valid) - 1; - } - while bits != 0 { - let lane = bits.trailing_zeros() as usize; - acc = acc.wrapping_add(values[base + lane] as i64); - bits &= bits - 1; - } - } - acc -} - -// ──────────────────────────────────────────────────────────────────────── -// Tests -// ──────────────────────────────────────────────────────────────────────── - -/// Sum a sub-word group field out of a **strided** record, over the records a -/// mask selects, widened to `i128` and range-checked into `i64`. -/// -/// The shape this exists for: a row-strided store whose each record carries a small -/// content-blind register, read under a runtime grouping — `groups × group_bytes` -/// little-endian fields per record. `lance-graph-java`'s V3 facet is the -/// motivating case (512-byte rows, a 12-byte register read as `6×2` / `4×3` / -/// `3×4`), but nothing here is specific to it. -/// -/// # Why this lives HERE -/// -/// It is the primitive a consumer would otherwise hand-roll with raw intrinsics, -/// which is exactly what the "all SIMD from `ndarray::simd`" invariant exists to -/// prevent. [`masked_sum_i32`] is contiguous `i32`; -/// [`eq_u32_strided_to_mask`] reads one aligned `u32` per record. Neither covers -/// "gather a sub-word group out of a strided register and widen-accumulate", so -/// the consumer had a real gap and this closes it. -/// -/// # Vectorisation, honestly -/// -/// **This kernel is scalar, and measurement is why — not oversight.** The access -/// pattern is one small register per record at a large stride (512 bytes in the -/// motivating case), so every record is on its own cache line and the loop is -/// memory-bound. The per-record work is 12 bytes; a vector register is 32-64. -/// There is no way to vector-load several records' registers at once because -/// they are not adjacent, and widening 6 `u16`s within one record does not fill -/// a lane. Vectorising the *decode* would optimise the part that is already -/// free. -/// -/// Should a caller ever present a CONTIGUOUS or small-stride variant, that is a -/// different primitive with a different name, and it would genuinely vectorise — -/// this one should not grow a flag for it. -/// -/// # Overflow -/// -/// Accumulates in `i128` and range-checks once, returning `None` rather than a -/// wrapped value. `i64` is not closed under this reduction: with -/// `group_bytes = 4` a single record contributes up to `groups × (2³² − 1)`. -/// -/// # Panics -/// -/// If `group_bytes` is not in `1..=4`, if `mask_words` is too short for -/// `n_records`, or if the last selected record's field would read past `bytes`. -/// Each is a caller contract violation rather than a recoverable condition. -/// -/// ``` -/// use ndarray::simd::masked_strided_group_sum; -/// -/// // Two 8-byte records; the register starts at byte 2 and holds 3 × u16 LE. -/// let mut b = vec![0u8; 16]; -/// b[2..8].copy_from_slice(&[1, 0, 2, 0, 3, 0]); // record 0 -> 1 + 2 + 3 -/// b[10..16].copy_from_slice(&[10, 0, 20, 0, 30, 0]); // record 1 -> 60 -/// // mask selects record 0 only -/// assert_eq!(masked_strided_group_sum(&b, 2, 8, 2, 3, 2, &[0b01]), Some(6)); -/// // both records -/// assert_eq!(masked_strided_group_sum(&b, 2, 8, 2, 3, 2, &[0b11]), Some(66)); -/// ``` -#[inline] -pub fn masked_strided_group_sum( - bytes: &[u8], first_offset: usize, stride_bytes: usize, n_records: usize, groups: usize, group_bytes: usize, - mask_words: &[u64], -) -> Option { - assert!((1..=4).contains(&group_bytes), "masked_strided_group_sum: group_bytes={group_bytes} outside 1..=4"); - let words = mask_words_for(n_records); - assert!( - mask_words.len() >= words, - "masked_strided_group_sum: mask_words.len()={} < required {}", - mask_words.len(), - words - ); - - let mut acc: i128 = 0; - for (w, &word) in mask_words.iter().take(words).enumerate() { - let base = w * 64; - if base >= n_records { - break; - } - let mut bits = word; - // Clamp the final partial word so a dirty tail cannot address a record - // that does not exist. Same guard, same reason, as `masked_sum_i32`. - let valid = n_records - base; - if valid < 64 { - bits &= (1u64 << valid) - 1; - } - while bits != 0 { - let rec = base + bits.trailing_zeros() as usize; - bits &= bits - 1; - let reg = rec * stride_bytes + first_offset; - let end = reg + groups * group_bytes; - assert!( - end <= bytes.len(), - "masked_strided_group_sum: record {rec} reads {reg}..{end}, past len {}", - bytes.len() - ); - for g in 0..groups { - let o = reg + g * group_bytes; - // Byte-wise, not a widened load: `o` is not guaranteed aligned - // for a 3-byte grouping, and an unaligned wide read is UB in - // Rust even where the hardware tolerates it. - let mut v: u32 = 0; - for k in 0..group_bytes { - v |= (bytes[o + k] as u32) << (8 * k); - } - acc += v as i128; - } - } - } - i64::try_from(acc).ok() -} - #[cfg(test)] mod tests { use super::*; @@ -1703,754 +917,6 @@ mod tests { assert!(out2.iter().all(|&v| v == 11), "batch_packed_i4_16 nibble=1+aux=10"); } - // ── Packed-bitmask predicates + mask algebra ──────────────────────────── - // - // Every test compares the shipped path against an INDEPENDENT scalar - // reference written inline here (never against the implementation's own - // scalar tail, which would be tautological), over a fixed-seed corpus plus - // the explicit edge cases: empty, 1, 63, 64, 65, non-multiples of 64, - // all-match, no-match, `u32::MAX` needle, `i32::MIN`/`i32::MAX` thresholds, - // and negative values. Bit order and the trailing-zero guarantee are - // asserted literally, against hand-computed `u64` words. - // - // Dispatch is compile-time, so one build exercises one backend; the - // scalar references below are what makes "all backends agree" checkable by - // re-running under `-Ctarget-cpu=x86-64-v3` (AVX2 arm) and - // `-Ctarget-cpu=x86-64-v4` (AVX-512 arm). - - /// Deterministic fixed-seed PRNG (SplitMix64) — no dev-dependency needed - /// and the corpus is byte-identical on every run and every backend. - fn splitmix64(state: &mut u64) -> u64 { - *state = state.wrapping_add(0x9E37_79B9_7F4A_7C15); - let mut z = *state; - z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); - z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); - z ^ (z >> 31) - } - - /// Independent reference: bit `i % 64` of word `i / 64` set where the - /// predicate holds, everything else zero. - fn ref_pack(values: &[T], n_words: usize, pred: impl Fn(T) -> bool) -> Vec { - let mut words = vec![0u64; n_words]; - for (i, &v) in values.iter().enumerate() { - if pred(v) { - words[i / 64] |= 1u64 << (i % 64); - } - } - words - } - - /// Lengths that straddle every boundary that matters: word edges (63/64/65), - /// the 16-lane group edge (15/16/17), and non-multiples of both. - const MASK_LENS: &[usize] = &[0, 1, 2, 15, 16, 17, 31, 32, 33, 47, 63, 64, 65, 100, 127, 128, 129, 200, 255, 256]; - - #[test] - fn eq_u32_to_mask_matches_scalar_reference() { - for &len in MASK_LENS { - let mut seed = 0xA5A5_1234_DEAD_BEEF; - let values: Vec = (0..len) - .map(|_| (splitmix64(&mut seed) % 7) as u32) - .collect(); - - for needle in [0u32, 1, 3, 6, 42, u32::MAX] { - let n_words = len.div_ceil(64); - let expected = ref_pack(&values, n_words, |v| v == needle); - - let mut got = vec![0u64; n_words]; - eq_u32_to_mask(&values, needle, &mut got); - assert_eq!(got, expected, "eq_u32_to_mask len={len} needle={needle}"); - } - } - } - - #[test] - fn eq_u32_to_mask_all_match_and_no_match() { - for &len in MASK_LENS { - let n_words = len.div_ceil(64); - - // All-match: every in-range bit set, every out-of-range bit clear. - let all = vec![9u32; len]; - let mut got = vec![0u64; n_words]; - eq_u32_to_mask(&all, 9, &mut got); - assert_eq!(got, ref_pack(&all, n_words, |v| v == 9), "all-match len={len}"); - // Independent cross-check on the count, so a wrong-but-consistent - // reference cannot hide: exactly `len` bits, no more. - let popcnt: u32 = got.iter().map(|w| w.count_ones()).sum(); - assert_eq!(popcnt as usize, len, "all-match popcount len={len}"); - - // No-match: strictly zero everywhere. - let mut got = vec![u64::MAX; n_words]; // pre-dirtied — must be overwritten - eq_u32_to_mask(&all, 10, &mut got); - assert!(got.iter().all(|&w| w == 0), "no-match must be all zeros, len={len}"); - } - } - - #[test] - fn eq_u32_to_mask_u32_max_needle_and_values() { - // u32::MAX is both a legal needle and a legal value; neither is special. - let values = [u32::MAX, 0, u32::MAX, 1, u32::MAX - 1]; - let mut got = [0u64; 1]; - eq_u32_to_mask(&values, u32::MAX, &mut got); - assert_eq!(got[0], 0b00101, "u32::MAX needle → bits 0 and 2"); - - eq_u32_to_mask(&values, u32::MAX - 1, &mut got); - assert_eq!(got[0], 0b10000, "u32::MAX-1 needle → bit 4 only"); - } - - /// The strided primitive against an independent reference, over an - /// AoS-facet buffer shape (u32 field at `first_offset` inside a - /// `stride_bytes`-wide row). Strides cover the contiguous case (4), a - /// facet within a 16-byte record, and a 512-byte row. - #[test] - fn eq_u32_strided_to_mask_matches_scalar_reference() { - for &count in MASK_LENS { - for &(first_offset, stride) in &[(0usize, 4usize), (4, 16), (16, 512), (0, 0)] { - let byte_len = if count == 0 { - 0 - } else { - first_offset + (count - 1) * stride + 4 - }; - let mut seed = 0x0F0F_CAFE_F00D_1234 ^ (stride as u64); - let mut bytes = vec![0u8; byte_len]; - // Fill every element position with a small-cardinality value so - // needles genuinely hit and miss. stride==0 has ONE position. - let positions = if stride == 0 { count.min(1) } else { count }; - let mut planted = Vec::with_capacity(positions); - for i in 0..positions { - let v = (splitmix64(&mut seed) % 5) as u32; - let off = first_offset + i * stride; - bytes[off..off + 4].copy_from_slice(&v.to_le_bytes()); - planted.push(v); - } - for needle in [0u32, 1, 4, 42] { - let n_words = count.div_ceil(64); - // Independent reference: read back the SAME strided walk - // scalar-only (stride 0 rereads element 0 `count` times). - let logical: Vec = (0..count) - .map(|i| { - if stride == 0 { - planted.first().copied().unwrap_or(0) - } else { - planted[i] - } - }) - .collect(); - let expected = ref_pack(&logical, n_words, |v| v == needle); - - let mut got = vec![u64::MAX; n_words]; // pre-dirtied - eq_u32_strided_to_mask(&bytes, first_offset, stride, count, needle, &mut got); - assert_eq!( - got, expected, - "strided eq count={count} off={first_offset} stride={stride} needle={needle}" - ); - } - } - } - } - - /// Parity with the contiguous primitive: stride 4 over the same values - /// must produce bit-identical masks — two independent implementations of - /// one specification. - #[test] - fn eq_u32_strided_stride4_matches_contiguous_primitive() { - for &count in MASK_LENS { - let mut seed = 0xBEE5_0000_0000_0001; - let values: Vec = (0..count) - .map(|_| (splitmix64(&mut seed) % 9) as u32) - .collect(); - let bytes: Vec = values.iter().flat_map(|v| v.to_le_bytes()).collect(); - let n_words = count.div_ceil(64); - let mut a = vec![0u64; n_words]; - let mut b = vec![0u64; n_words]; - for needle in [0u32, 3, 8, u32::MAX] { - eq_u32_to_mask(&values, needle, &mut a); - eq_u32_strided_to_mask(&bytes, 0, 4, count, needle, &mut b); - assert_eq!(a, b, "contiguous vs strided count={count} needle={needle}"); - } - } - } - - #[test] - #[should_panic(expected = "out of bounds")] - fn eq_u32_strided_rejects_a_last_element_past_the_buffer() { - // 3 elements at stride 16 need bytes 32..36; a 35-byte buffer is short. - let bytes = vec![0u8; 35]; - let mut words = [0u64; 1]; - eq_u32_strided_to_mask(&bytes, 0, 16, 3, 7, &mut words); - } - - #[test] - #[should_panic(expected = "offset arithmetic overflow")] - fn eq_u32_strided_rejects_overflowing_offset_arithmetic() { - let bytes = vec![0u8; 64]; - let mut words = [0u64; 1]; - // (count-1) * stride overflows usize — must panic, not wrap into a - // bogus in-bounds read. - eq_u32_strided_to_mask(&bytes, 0, usize::MAX, 3, 7, &mut words); - } - - #[test] - fn eq_u32_strided_empty_count_writes_only_zeros() { - let bytes: Vec = Vec::new(); - let mut words = [u64::MAX; 2]; - eq_u32_strided_to_mask(&bytes, 0, 512, 0, 7, &mut words); - assert_eq!(words, [0, 0], "count=0 must still overwrite the destination"); - } - - #[test] - fn gt_i32_to_mask_matches_scalar_reference() { - for &len in MASK_LENS { - let mut seed = 0x1357_9BDF_0246_8ACE; - // Full signed spread including both extremes, seeded deterministically. - let values: Vec = (0..len) - .map(|i| match i % 11 { - 0 => i32::MIN, - 1 => i32::MAX, - 2 => 0, - 3 => -1, - 4 => 1, - _ => splitmix64(&mut seed) as i32, - }) - .collect(); - - for threshold in [i32::MIN, i32::MIN + 1, -1000, -1, 0, 1, 1000, i32::MAX - 1, i32::MAX] { - let n_words = len.div_ceil(64); - let expected = ref_pack(&values, n_words, |v| v > threshold); - - let mut got = vec![0u64; n_words]; - gt_i32_to_mask(&values, threshold, &mut got); - assert_eq!(got, expected, "gt_i32_to_mask len={len} threshold={threshold}"); - } - } - } - - #[test] - fn gt_i32_to_mask_signed_not_bitwise() { - // The trap: -1 as a bit pattern (0xFFFF_FFFF) is greater than 0 - // unsigned, but -1 > 0 is false. A backend that packed an unsigned - // compare would set bit 1 here. - let values = [5i32, -1, 0, -2_000_000_000, 2_000_000_000]; - let mut got = [0u64; 1]; - gt_i32_to_mask(&values, 0, &mut got); - assert_eq!(got[0], 0b10001, "only +5 and +2e9 exceed 0"); - } - - #[test] - fn gt_i32_to_mask_threshold_extremes() { - let values = [i32::MIN, i32::MIN + 1, 0, i32::MAX - 1, i32::MAX]; - let mut got = [0u64; 1]; - - // i32::MIN threshold: everything strictly greater — all but lane 0. - gt_i32_to_mask(&values, i32::MIN, &mut got); - assert_eq!(got[0], 0b11110, "i32::MIN threshold excludes only i32::MIN itself"); - - // i32::MAX threshold: nothing exceeds it, and `>` is strict so the - // i32::MAX lane itself is clear too. - got[0] = u64::MAX; - gt_i32_to_mask(&values, i32::MAX, &mut got); - assert_eq!(got[0], 0, "nothing exceeds i32::MAX"); - - // i32::MAX - 1 threshold: only i32::MAX. - gt_i32_to_mask(&values, i32::MAX - 1, &mut got); - assert_eq!(got[0], 0b10000, "only i32::MAX exceeds i32::MAX-1"); - } - - /// The real correctness trap: bits past `values.len()` in the last word. - /// A stale high bit would silently inflate every downstream popcount. - #[test] - fn trailing_bits_beyond_len_are_zero() { - for &len in &[1usize, 15, 16, 17, 33, 63, 65, 100, 127, 129, 200] { - let n_words = len.div_ceil(64); - let used = len % 64; // 0 ⇒ the final word is entirely in range - - // Every element matches, so ONLY the out-of-range bits can be zero. - let u = vec![1u32; len]; - let mut got = vec![u64::MAX; n_words + 2]; // pre-dirtied, plus surplus words - eq_u32_to_mask(&u, 1, &mut got); - if used != 0 { - let expected_last = (1u64 << used) - 1; - assert_eq!(got[n_words - 1], expected_last, "eq trailing bits len={len}"); - } else { - assert_eq!(got[n_words - 1], u64::MAX, "eq full final word len={len}"); - } - assert!(got[n_words..].iter().all(|&w| w == 0), "eq surplus words must be zeroed, len={len}"); - - let i = vec![1i32; len]; - let mut got = vec![u64::MAX; n_words + 2]; - gt_i32_to_mask(&i, 0, &mut got); - if used != 0 { - let expected_last = (1u64 << used) - 1; - assert_eq!(got[n_words - 1], expected_last, "gt trailing bits len={len}"); - } else { - assert_eq!(got[n_words - 1], u64::MAX, "gt full final word len={len}"); - } - assert!(got[n_words..].iter().all(|&w| w == 0), "gt surplus words must be zeroed, len={len}"); - } - } - - #[test] - fn empty_input_writes_only_zeros() { - let mut got = [u64::MAX; 3]; - eq_u32_to_mask(&[], 7, &mut got); - assert_eq!(got, [0u64; 3], "empty eq"); - - let mut got = [u64::MAX; 3]; - gt_i32_to_mask(&[], 7, &mut got); - assert_eq!(got, [0u64; 3], "empty gt"); - - // Zero-length destination is legal for a zero-length input. - eq_u32_to_mask(&[], 7, &mut []); - gt_i32_to_mask(&[], 7, &mut []); - - assert_eq!(masked_sum_i32(&[], &[]), 0, "empty masked_sum"); - } - - #[test] - fn single_element_lands_in_bit_zero() { - let mut got = [u64::MAX; 1]; - eq_u32_to_mask(&[7u32], 7, &mut got); - assert_eq!(got[0], 1, "one matching element ⇒ exactly bit 0"); - eq_u32_to_mask(&[8u32], 7, &mut got); - assert_eq!(got[0], 0, "one non-matching element ⇒ no bits"); - } - - /// Bit order asserted against hand-computed literals — the one test that - /// would catch an MSB-first or word-swapped backend, which a - /// reference-vs-implementation comparison alone cannot (both could be - /// wrong the same way if the reference were derived from the code). - #[test] - fn bit_order_is_lsb_first_within_each_word() { - // 130 elements: matches at 0, 1, 63 (word 0 low + high edge), - // 64, 65, 127 (word 1), and 128 (word 2 bit 0). - let matching = [0usize, 1, 63, 64, 65, 127, 128]; - let mut values = vec![0u32; 130]; - for &i in &matching { - values[i] = 1; - } - - let mut got = [0u64; 3]; - eq_u32_to_mask(&values, 1, &mut got); - - assert_eq!(got[0], (1u64 << 0) | (1u64 << 1) | (1u64 << 63), "word 0: elements 0, 1, 63"); - assert_eq!(got[1], (1u64 << 0) | (1u64 << 1) | (1u64 << 63), "word 1: elements 64, 65, 127 → bits 0, 1, 63"); - assert_eq!(got[2], 1u64 << 0, "word 2: element 128 → bit 0, rest zero"); - - // Element 64 is bit 0 of word 1, NOT bit 64-of-something or the high - // bit of word 0 — the word-boundary claim, stated as its own literal. - let mut only_64 = vec![0u32; 130]; - only_64[64] = 1; - let mut got = [0u64; 3]; - eq_u32_to_mask(&only_64, 1, &mut got); - assert_eq!(got, [0u64, 1u64, 0u64], "element 64 ⇒ word 1 bit 0 alone"); - } - - // ── mask algebra ──────────────────────────────────────────────────────── - - #[test] - fn mask_and_or_match_scalar_reference() { - // Lengths straddling the 8-word U64x8 group boundary. - for &len in &[0usize, 1, 2, 7, 8, 9, 15, 16, 17, 31, 63, 64, 100] { - let mut seed = 0xFEED_FACE_CAFE_0001; - let a: Vec = (0..len).map(|_| splitmix64(&mut seed)).collect(); - let b: Vec = (0..len).map(|_| splitmix64(&mut seed)).collect(); - - let ref_and: Vec = a.iter().zip(&b).map(|(x, y)| x & y).collect(); - let ref_or: Vec = a.iter().zip(&b).map(|(x, y)| x | y).collect(); - - let mut dst = vec![0xDEAD_BEEFu64; len]; - mask_and(&a, &b, &mut dst); - assert_eq!(dst, ref_and, "mask_and len={len}"); - - let mut dst = vec![0xDEAD_BEEFu64; len]; - mask_or(&a, &b, &mut dst); - assert_eq!(dst, ref_or, "mask_or len={len}"); - - let mut dst = a.clone(); - mask_and_assign(&mut dst, &b); - assert_eq!(dst, ref_and, "mask_and_assign len={len}"); - - let mut dst = a.clone(); - mask_or_assign(&mut dst, &b); - assert_eq!(dst, ref_or, "mask_or_assign len={len}"); - } - } - - #[test] - fn mask_algebra_identities() { - let a = vec![0x0F0F_0F0F_0F0F_0F0Fu64; 20]; - let zeros = vec![0u64; 20]; - let ones = vec![u64::MAX; 20]; - - let mut dst = vec![1u64; 20]; - mask_and(&a, &ones, &mut dst); - assert_eq!(dst, a, "x & ALL == x"); - - mask_and(&a, &zeros, &mut dst); - assert_eq!(dst, zeros, "x & 0 == 0"); - - mask_or(&a, &zeros, &mut dst); - assert_eq!(dst, a, "x | 0 == x"); - - mask_or(&a, &ones, &mut dst); - assert_eq!(dst, ones, "x | ALL == ALL"); - - // Narrowing: AND is monotone, so the popcount can only shrink. - let mut seed = 0x0BAD_C0DE_0BAD_C0DE; - let b: Vec = (0..20).map(|_| splitmix64(&mut seed)).collect(); - let mut dst = vec![0u64; 20]; - mask_and(&a, &b, &mut dst); - let pc = |w: &[u64]| -> u32 { w.iter().map(|x| x.count_ones()).sum() }; - assert!(pc(&dst) <= pc(&a), "AND narrows"); - assert!(pc(&dst) <= pc(&b), "AND narrows"); - // ...and non-trivially so, or the assertion above is vacuous. - assert!(pc(&dst) < pc(&a), "AND must actually remove bits on this corpus"); - } - - #[test] - #[should_panic(expected = "length mismatch")] - fn mask_and_rejects_length_mismatch() { - let mut dst = [0u64; 4]; - mask_and(&[0u64; 4], &[0u64; 3], &mut dst); - } - - // ── mask_andnot (a & !b) ───────────────────────────────────────────────── - - #[test] - fn mask_andnot_matches_scalar_reference() { - // Same length set as `mask_and_or_match_scalar_reference`, straddling - // the 8-word U64x8 group boundary; len=2 is the `mask_words_for(70)` - // shape (70 rows -> 2 words, a 6-bit tail in the second word). - for &len in &[0usize, 1, 2, 7, 8, 9, 15, 16, 17, 31, 63, 64, 100] { - let mut seed = 0xA11C_E5EE_D000_0001; - let a: Vec = (0..len).map(|_| splitmix64(&mut seed)).collect(); - let b: Vec = (0..len).map(|_| splitmix64(&mut seed)).collect(); - - let ref_andnot: Vec = a.iter().zip(&b).map(|(x, y)| x & !y).collect(); - - let mut dst = vec![0xDEAD_BEEFu64; len]; - mask_andnot(&a, &b, &mut dst); - assert_eq!(dst, ref_andnot, "mask_andnot len={len}"); - - let mut dst = a.clone(); - mask_andnot_assign(&mut dst, &b); - assert_eq!(dst, ref_andnot, "mask_andnot_assign len={len}"); - } - } - - #[test] - fn mask_andnot_algebra_identities() { - let mut seed = 0x1357_9BDF_2468_ACE0; - let a: Vec = (0..20).map(|_| splitmix64(&mut seed)).collect(); - let b: Vec = (0..20).map(|_| splitmix64(&mut seed)).collect(); - - // (a & !b) | (a & b) == a — partitioning a's bits by whether b also - // has them set recovers a exactly. - let mut a_andnot_b = vec![0u64; 20]; - mask_andnot(&a, &b, &mut a_andnot_b); - let mut a_and_b = vec![0u64; 20]; - mask_and(&a, &b, &mut a_and_b); - let mut recombined = vec![0u64; 20]; - mask_or(&a_andnot_b, &a_and_b, &mut recombined); - assert_eq!(recombined, a, "(a & !b) | (a & b) == a"); - - // (a & !b) & b == 0 — the "not b" half can never overlap b. - let mut overlap = vec![0u64; 20]; - mask_and(&a_andnot_b, &b, &mut overlap); - assert_eq!(overlap, vec![0u64; 20], "(a & !b) & b == 0"); - - // ...and non-trivially so: on this corpus a_andnot_b must actually - // differ from a (b removes real bits), or both identities above hold - // vacuously of a no-op. - assert_ne!(a_andnot_b, a, "andnot must actually remove bits on this corpus"); - } - - #[test] - fn mask_andnot_preserves_conforming_tail() { - // 2 words = the `mask_words_for(70)` shape: word 0 fully valid (rows - // 0..63), word 1 valid only in its low 7 bits (rows 64..70); the - // tail is word 1 bits 7..63, which a conforming mask always holds - // zero. - const TAIL_MASK: u64 = !0x7Fu64; // bits 7..63 - - // Arm 1: a conforms (tail zero), b is maximally non-conforming (all - // bits set, including its own tail) — dst must still be zero - // everywhere, tail included, because `a & !b` can never exceed `a`. - let a = [0x1234_5678_9ABC_DEF0u64, 0x0000_0000_0000_005Bu64]; - assert_eq!(a[1] & TAIL_MASK, 0, "fixture precondition: a's tail is zero"); - let b = [u64::MAX; 2]; - let mut dst = [0xDEAD_BEEFu64; 2]; - mask_andnot(&a, &b, &mut dst); - assert_eq!(dst, [0u64, 0u64], "a & !(all-ones) == 0, tail included"); - - // Arm 2: a still conforms; b's body is zero (so it removes nothing - // from a) but b's tail is dirty (all ones) — exactly the shape where - // `!b` flips a normally-zero tail to all ones. dst must equal a - // exactly, and in particular dst's tail must stay zero: a's tail was - // already zero, and `a & !b` can only ever narrow a, never widen it. - let b_dirty_tail = [0u64, TAIL_MASK]; - assert_ne!(b_dirty_tail[1] & TAIL_MASK, 0, "fixture precondition: b's tail is dirty"); - let mut dst = [0xDEAD_BEEFu64; 2]; - mask_andnot(&a, &b_dirty_tail, &mut dst); - assert_eq!(dst, a, "a & !b == a when b's body is 0, even with a dirty b tail"); - assert_eq!(dst[1] & TAIL_MASK, 0, "dst's tail stays zero despite b's dirty tail"); - } - - #[test] - #[should_panic(expected = "length mismatch")] - fn mask_andnot_rejects_length_mismatch() { - let mut dst = [0u64; 4]; - mask_andnot(&[0u64; 4], &[0u64; 3], &mut dst); - } - - #[test] - #[should_panic(expected = "length mismatch")] - fn mask_andnot_assign_rejects_length_mismatch() { - let mut a = [0u64; 4]; - mask_andnot_assign(&mut a, &[0u64; 3]); - } - - // ── mask_ternlog (any 3-input Boolean, one pass) ───────────────────────── - - /// Truth-table reference evaluated one BIT at a time — independent of - /// both `ternlog_word` (bit-parallel over rows) and every backend lane. - fn ref_ternlog_bitwise(a: u64, b: u64, c: u64, imm: i32) -> u64 { - let mut r = 0u64; - for bit in 0..64 { - let idx = (((a >> bit) & 1) << 2) | (((b >> bit) & 1) << 1) | ((c >> bit) & 1); - if (imm >> idx) & 1 == 1 { - r |= 1u64 << bit; - } - } - r - } - - /// Exercise one IMM over the family's standard length set, both forms, - /// against the bit-serial reference. - fn check_ternlog_imm() { - for &len in &[0usize, 1, 2, 7, 8, 9, 15, 16, 17, 31, 63, 64, 100] { - let mut seed = 0x7E12_10C0_0000_0001 ^ (IMM as u64); - let a: Vec = (0..len).map(|_| splitmix64(&mut seed)).collect(); - let b: Vec = (0..len).map(|_| splitmix64(&mut seed)).collect(); - let c: Vec = (0..len).map(|_| splitmix64(&mut seed)).collect(); - let expect: Vec = (0..len) - .map(|i| ref_ternlog_bitwise(a[i], b[i], c[i], IMM)) - .collect(); - - let mut dst = vec![0xDEAD_BEEFu64; len]; - mask_ternlog::(&a, &b, &c, &mut dst); - assert_eq!(dst, expect, "mask_ternlog imm={IMM:#04x} len={len}"); - - let mut dst = a.clone(); - mask_ternlog_assign::(&mut dst, &b, &c); - assert_eq!(dst, expect, "mask_ternlog_assign imm={IMM:#04x} len={len}"); - } - } - - #[test] - fn mask_ternlog_matches_bitwise_reference_for_all_256_tables() { - // Const generics need a literal per instantiation; a macro unrolls - // all 256 so no table is left to "obviously the same as the others". - macro_rules! all_imms { - ($($imm:literal),* $(,)?) => { $( check_ternlog_imm::<$imm>(); )* }; - } - all_imms!( - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, 0x11, - 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x1F, 0x20, 0x21, 0x22, 0x23, - 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2A, 0x2B, 0x2C, 0x2D, 0x2E, 0x2F, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, - 0x36, 0x37, 0x38, 0x39, 0x3A, 0x3B, 0x3C, 0x3D, 0x3E, 0x3F, 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, - 0x48, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, - 0x5A, 0x5B, 0x5C, 0x5D, 0x5E, 0x5F, 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6A, 0x6B, - 0x6C, 0x6D, 0x6E, 0x6F, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, 0x7B, 0x7C, 0x7D, - 0x7E, 0x7F, 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8A, 0x8B, 0x8C, 0x8D, 0x8E, 0x8F, - 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9A, 0x9B, 0x9C, 0x9D, 0x9E, 0x9F, 0xA0, 0xA1, - 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0xAA, 0xAB, 0xAC, 0xAD, 0xAE, 0xAF, 0xB0, 0xB1, 0xB2, 0xB3, - 0xB4, 0xB5, 0xB6, 0xB7, 0xB8, 0xB9, 0xBA, 0xBB, 0xBC, 0xBD, 0xBE, 0xBF, 0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, - 0xC6, 0xC7, 0xC8, 0xC9, 0xCA, 0xCB, 0xCC, 0xCD, 0xCE, 0xCF, 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, - 0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF, 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 0xE8, 0xE9, - 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0xFA, 0xFB, - 0xFC, 0xFD, 0xFE, 0xFF, - ); - } - - #[test] - fn mask_ternlog_and3_equals_two_and_passes() { - // The consumer motivation: `selected & src & gate` as one AND3 pass - // must be bit-identical to the two-pass `mask_and_assign` spelling. - use crate::simd::ternlog::AND3; - let mut seed = 0xA3D3_0000_0000_0001; - let sel: Vec = (0..20).map(|_| splitmix64(&mut seed)).collect(); - let src: Vec = (0..20).map(|_| splitmix64(&mut seed)).collect(); - let gate: Vec = (0..20).map(|_| splitmix64(&mut seed)).collect(); - - let mut two_pass = sel.clone(); - mask_and_assign(&mut two_pass, &src); - mask_and_assign(&mut two_pass, &gate); - - let mut one_pass = sel.clone(); - mask_ternlog_assign::(&mut one_pass, &src, &gate); - assert_eq!(one_pass, two_pass, "AND3 == and∘and"); - - // Non-vacuous: the narrowing must have removed bits, and both narrower - // operands must have contributed (each alone leaves a different set). - assert_ne!(one_pass, sel, "AND3 must narrow on this corpus"); - let mut src_only = sel.clone(); - mask_and_assign(&mut src_only, &src); - assert_ne!(one_pass, src_only, "gate must contribute, not just src"); - } - - #[test] - fn mask_ternlog_tail_conforms_iff_imm_is_even() { - // 2 words = the `mask_words_for(70)` shape; tail = word 1 bits 7..63. - const TAIL_MASK: u64 = !0x7Fu64; - let a = [0x1234_5678_9ABC_DEF0u64, 0x0000_0000_0000_005Bu64]; - let b = [0x0F0F_0F0F_0F0F_0F0Fu64, 0x0000_0000_0000_0071u64]; - let c = [0xFFFF_0000_FFFF_0000u64, 0x0000_0000_0000_002Eu64]; - for m in [&a, &b, &c] { - assert_eq!(m[1] & TAIL_MASK, 0, "fixture precondition: conforming inputs"); - } - use crate::simd::ternlog::{AND3, MAJ3, OR3, XOR3}; - - // Every named table is even: tail stays zero. - let mut d = [0xDEAD_BEEFu64; 2]; - mask_ternlog::(&a, &b, &c, &mut d); - assert_eq!(d[1] & TAIL_MASK, 0, "AND3 tail"); - mask_ternlog::(&a, &b, &c, &mut d); - assert_eq!(d[1] & TAIL_MASK, 0, "OR3 tail"); - mask_ternlog::(&a, &b, &c, &mut d); - assert_eq!(d[1] & TAIL_MASK, 0, "MAJ3 tail"); - mask_ternlog::(&a, &b, &c, &mut d); - assert_eq!(d[1] & TAIL_MASK, 0, "XOR3 tail"); - - // The can-it-fire half: an ODD table (NOR3 = 0x01, true of all-zero - // inputs) sets every tail bit, so the doc's "iff even" is a real - // boundary and not a restatement of the inputs. - mask_ternlog::<0x01>(&a, &b, &c, &mut d); - assert_eq!(d[1] & TAIL_MASK, TAIL_MASK, "odd IMM fills the tail"); - - // Subset-shaped table against dirty b/c: still a subset of a. - let dirty = [u64::MAX; 2]; - mask_ternlog::(&a, &dirty, &dirty, &mut d); - assert_eq!(d, a, "AND3 against all-ones is a"); - assert_eq!(d[1] & TAIL_MASK, 0, "AND3 tail follows a's tail"); - } - - #[test] - #[should_panic(expected = "length mismatch")] - fn mask_ternlog_rejects_length_mismatch() { - let mut dst = [0u64; 4]; - mask_ternlog::<0x80>(&[0u64; 4], &[0u64; 4], &[0u64; 3], &mut dst); - } - - #[test] - #[should_panic(expected = "length mismatch")] - fn mask_ternlog_assign_rejects_length_mismatch() { - let mut a = [0u64; 4]; - mask_ternlog_assign::<0x80>(&mut a, &[0u64; 3], &[0u64; 4]); - } - - #[test] - #[should_panic(expected = "out_words.len()")] - fn eq_u32_to_mask_rejects_short_destination() { - // 65 elements need 2 words; 1 must be refused, not silently truncated. - let values = vec![0u32; 65]; - let mut got = [0u64; 1]; - eq_u32_to_mask(&values, 0, &mut got); - } - - // ── masked_sum_i32 ────────────────────────────────────────────────────── - - #[test] - fn masked_sum_i32_matches_scalar_reference() { - for &len in MASK_LENS { - let mut seed = 0x2468_ACE0_1357_9BDF; - let values: Vec = (0..len).map(|_| splitmix64(&mut seed) as i32).collect(); - let n_words = len.div_ceil(64); - - for pattern in [0u64, u64::MAX, 0x5555_5555_5555_5555, 0xAAAA_AAAA_AAAA_AAAA, 1] { - let mask = vec![pattern; n_words]; - // Independent reference: widen every selected element to i64. - let expected: i64 = values - .iter() - .enumerate() - .filter(|(i, _)| mask[i / 64] >> (i % 64) & 1 == 1) - .map(|(_, &v)| v as i64) - .sum(); - let got = masked_sum_i32(&values, &mask); - assert_eq!(got, expected, "masked_sum_i32 len={len} pattern={pattern:#x}"); - } - } - } - - #[test] - fn masked_sum_i32_widens_beyond_i32_range() { - // 64 × i32::MAX = 137_438_953_408, which overflows i32 by ~64×. An - // implementation that reduced in i32 (e.g. `I32x16::reduce_sum`) would - // wrap here; the widened contract says it must not. - let values = [i32::MAX; 64]; - let got = masked_sum_i32(&values, &[u64::MAX]); - assert_eq!(got, 64 * i32::MAX as i64); - assert!(got > i32::MAX as i64, "result genuinely exceeds i32 range"); - - // Same on the negative side. - let values = [i32::MIN; 64]; - let got = masked_sum_i32(&values, &[u64::MAX]); - assert_eq!(got, 64 * i32::MIN as i64); - assert!(got < i32::MIN as i64); - } - - #[test] - fn masked_sum_i32_ignores_bits_past_len() { - // 3 elements, an all-ones mask word: bits 3..63 must be ignored, not - // used to index past the slice (which would panic) or counted. - let values = [10i32, 20, 30]; - assert_eq!(masked_sum_i32(&values, &[u64::MAX]), 60); - - // Same across a word boundary: 65 elements, both words all-ones. - let values: Vec = (0..65).collect(); - let expected: i64 = (0..65i64).sum(); - assert_eq!(masked_sum_i32(&values, &[u64::MAX; 2]), expected); - } - - #[test] - fn masked_sum_i32_empty_mask_is_zero() { - let values: Vec = (1..=100).collect(); - assert_eq!(masked_sum_i32(&values, &[0u64; 2]), 0, "no bits set ⇒ 0"); - } - - /// End-to-end composition: the shape the ABI's fused plan runs — two - /// predicates ANDed, then counted and summed. Ties the seven primitives - /// plus `popcount_batch_u64` together on one corpus. - #[test] - fn predicates_compose_into_count_and_sum() { - const N: usize = 1000; - let classes: Vec = (0..N).map(|i| (i % 4) as u32).collect(); - let values: Vec = (0..N).map(|i| i as i32 - 500).collect(); - let n_words = N.div_ceil(64); - - let mut m_class = vec![0u64; n_words]; - eq_u32_to_mask(&classes, 2, &mut m_class); - let mut m_value = vec![0u64; n_words]; - gt_i32_to_mask(&values, 0, &mut m_value); - - let mut acc = vec![u64::MAX; n_words]; - mask_and_assign(&mut acc, &m_class); - mask_and_assign(&mut acc, &m_value); - - // Independent reference over the same predicates. - let want: Vec = (0..N) - .filter(|&i| classes[i] == 2 && values[i] > 0) - .collect(); - let count = crate::bitwise::popcount_batch_u64(&acc); - assert_eq!(count as usize, want.len(), "fused count"); - let sum_ref: i64 = want.iter().map(|&i| values[i] as i64).sum(); - assert_eq!(masked_sum_i32(&values, &acc), sum_ref, "fused sum"); - - // Anti-vacuity: the composition must actually narrow, or this test - // would pass for a no-op AND. `acc` starts as all N rows. - assert!(count > 0, "the fused predicate must select something"); - assert!((count as usize) < N / 4, "the fused predicate must be strictly narrower than either operand"); - } - /// Exercises the AMX dispatch tier added on top of `gemm_u8_i8`'s /// compile-time cascade. On AMX-enabled silicon (Sapphire Rapids+ /// with the right OS prctl), 16/16/64-aligned shapes go through @@ -2471,91 +937,4 @@ mod tests { gemm_u8_i8(&a, &b, &mut c, m, n, k); assert_eq!(c, expected, "gemm_u8_i8 AMX path mismatch"); } - - // ── masked_strided_group_sum ── - - /// The three groupings of a 12-byte register read the SAME bytes and must - /// give three DIFFERENT answers — otherwise every test below would pass for - /// an implementation that ignored `groups`/`group_bytes`. - #[test] - fn each_grouping_of_the_same_register_reads_it_differently() { - let mut b = vec![0u8; 512]; - for k in 0..12 { - b[4 + k] = (k + 1) as u8; - } - let m = [0b1u64]; - let rails = masked_strided_group_sum(&b, 4, 512, 1, 6, 2, &m).unwrap(); - let trips = masked_strided_group_sum(&b, 4, 512, 1, 4, 3, &m).unwrap(); - let quads = masked_strided_group_sum(&b, 4, 512, 1, 3, 4, &m).unwrap(); - - // Hand-computed from bytes 1..=12, little-endian per group. - assert_eq!(rails, 0x0201 + 0x0403 + 0x0605 + 0x0807 + 0x0A09 + 0x0C0B); - assert_eq!(trips, 0x030201 + 0x060504 + 0x090807 + 0x0C0B0A); - assert_eq!(quads, 0x04030201 + 0x08070605 + 0x0C0B0A09); - assert!(rails != trips && trips != quads && rails != quads); - } - - /// The mask selects records rather than being decoration, and the stride is - /// respected: two records with different content must sum separately and - /// additively. - #[test] - fn the_mask_and_the_stride_both_bind() { - let mut b = vec![0u8; 2 * 64]; - b[0..4].copy_from_slice(&[1, 0, 2, 0]); - b[64..68].copy_from_slice(&[10, 0, 20, 0]); - let f = |m: u64| masked_strided_group_sum(&b, 0, 64, 2, 2, 2, &[m]).unwrap(); - assert_eq!(f(0b00), 0, "an empty mask sums nothing"); - assert_eq!(f(0b01), 3); - assert_eq!(f(0b10), 30); - assert_eq!(f(0b11), 33, "additive over disjoint selections"); - } - - /// A dirty tail bit past `n_records` is ignored rather than read — the - /// buffer here is too short for it, so an unclamped kernel would panic. - #[test] - fn a_dirty_tail_bit_is_ignored() { - let mut b = vec![0u8; 2 * 16]; - b[0..2].copy_from_slice(&[5, 0]); - b[16..18].copy_from_slice(&[7, 0]); - let clean = masked_strided_group_sum(&b, 0, 16, 2, 1, 2, &[0b11]).unwrap(); - let dirty = masked_strided_group_sum(&b, 0, 16, 2, 1, 2, &[0b1111]).unwrap(); - assert_eq!(clean, 12); - assert_eq!(clean, dirty); - } - - /// Overflow is reported, not wrapped. Four max-valued u32 groups per record - /// over many records exceeds `i64::MAX`; the boundary itself is asserted so - /// the claim is checkable rather than narrated. - #[test] - fn overflow_is_reported_rather_than_wrapped() { - let recs = 8usize; - let mut b = vec![0xFFu8; recs * 16]; - let m = [0xFFu64]; - // Small case: comfortably inside i64. - let small = masked_strided_group_sum(&b, 0, 16, recs, 3, 4, &m).unwrap(); - assert_eq!(small, recs as i64 * 3 * 0xFFFF_FFFF); - - // The documented bound, checked: how many max quad records fit? - let per_record = 3i128 * 0xFFFF_FFFFi128; - assert_eq!(i64::MAX as i128 / per_record, 715_827_882); - - // And the range check itself is what decides, not a wrap. - assert!(i64::try_from(i64::MAX as i128 + 1).is_err()); - b.clear(); - } - - #[test] - #[should_panic(expected = "group_bytes")] - fn a_group_wider_than_four_bytes_is_rejected() { - let b = vec![0u8; 64]; - let _ = masked_strided_group_sum(&b, 0, 16, 1, 1, 5, &[0b1]); - } - - #[test] - #[should_panic(expected = "past len")] - fn a_record_reading_past_the_buffer_is_rejected() { - let b = vec![0u8; 8]; - // Record 0's register would read 0..12 out of an 8-byte buffer. - let _ = masked_strided_group_sum(&b, 0, 16, 1, 3, 4, &[0b1]); - } } diff --git a/src/simd_masking_ops.rs b/src/simd_masking_ops.rs new file mode 100644 index 00000000..6e1183f3 --- /dev/null +++ b/src/simd_masking_ops.rs @@ -0,0 +1,2730 @@ +//! Packed-bitmask predicates, mask algebra, and masked reductions — the +//! **ergonomic masking layer** of the SIMD stack. +//! +//! # Where this sits (the three-layer contract, operator-ruled 2026-09-13) +//! +//! ```text +//! consumers (lance-graph-mask-risc, lgj-abi kernels, planner) +//! │ semantic ops: TERNLOG, AND, XOR, COUNT, eq→mask … +//! ▼ +//! simd_masking_ops.rs ← THIS FILE: slice/chunk/tail ergonomics, in-place +//! │ forms, mask composition, masked reductions +//! ▼ +//! simd.rs architecture-agnostic lane types (U64x8, U32x16, …) +//! │ compile-time backend selection +//! ▼ +//! simd_{avx512,avx2,neon,wasm,scalar}.rs each owns its realization +//! ``` +//! +//! The rule that keeps the layers honest: **no backend semantics live here.** +//! This file composes lane-level primitives (`U32x16::eq_bitmask`, +//! `U64x8::ternlog::`, `I32x16::gt_bitmask`, …) into slice-level +//! machinery — it never branches on an ISA, never names an intrinsic, and +//! never carries a per-architecture cost model. What `U64x8::ternlog` *is* on +//! AVX2 versus AVX-512 versus NEON is entirely the corresponding backend +//! file's business. Conversely, chunking, tail handling, reusable-destination +//! (`*_assign`) forms, and fused convenience compositions belong HERE and +//! never in a backend. +//! +//! Sibling of [`crate::simd_int_ops`] (integer arithmetic / conversion), split +//! out so masking is a first-class execution family rather than a collection +//! of functions that accumulated inside integer ops. Every `pub fn` is +//! re-exported through [`crate::simd`]; consumers import from there. +//! +//! # Bit order (NORMATIVE) +//! +//! Element `i` lives at bit `i % 64` of word `i / 64`, LSB-first; every bit +//! at or past the element count is zero. The full statement, and why it is +//! structural rather than a tail special case, is in the section header below. +#![forbid(unsafe_code)] + +// ──────────────────────────────────────────────────────────────────────── +// Packed-bitmask predicates + mask algebra (the columnar-selection lane) +// ──────────────────────────────────────────────────────────────────────── +// +// These seven primitives are the vector half of a columnar filter: turn a +// lane of values into a packed bit-per-row mask, compose masks with boolean +// algebra, and reduce a value lane under a mask. They are the substrate the +// `lance-graph-java` ABI membrane rides (`lgj_op_eq_u32`, `lgj_op_gt_i32`, +// `lgj_mask_and`, `lgj_mask_or`, `lgj_plan_eval`, `lgj_reduce_sum_i32`), and +// the reason that membrane needs no SIMD of its own — a consumer crate that +// wrote its own compare-and-pack loop would be an `ndarray::simd` bypass. +// +// ## Bit order (NORMATIVE — every function below obeys it) +// +// Element index `i` lives at **bit `i % 64` of word `i / 64`**; LSB-first +// within each word, so element 0 is bit 0 of `out_words[0]` and element 64 is +// bit 0 of `out_words[1]`. This matches the `MASK_WORD` lane definition on +// the ABI side ("a `u64` of 64 packed row bits, LSB = lowest row index") and +// the lane-level `u16` convention already established by +// `I32x16::cmpge_zero_mask`. +// +// **Trailing bits beyond `values.len()` in the final word are always written +// as 0**, as are any surplus words in a longer-than-necessary `out_words`. +// This is load-bearing: those bits feed straight into `popcount_batch_u64`, +// so a stale high bit would silently inflate a count. Every writer below +// zeroes the whole destination first and then only ever sets bits for +// in-range elements, which makes the guarantee structural rather than a +// tail-handling special case that could be forgotten. +// +// ## Why free functions here, not methods on a wrapper +// +// The W1a consumer contract's "struct method, not free function" litmus +// governs **lane-level** primitives, where a free function fragments the +// typed-wrapper surface. These are **slice-level**, the same tier as +// `add_i8` / `dot_i8` / `min_i8` above, and they are built *on* lane methods +// (`U32x16::eq_bitmask`, `I32x16::gt_bitmask`) that do live on the wrappers. + +/// Number of packed mask words needed to cover `n` elements. +#[inline(always)] +fn mask_words_for(n: usize) -> usize { + n.div_ceil(64) +} + +// Every contiguous LANE op below — the predicate builders over `&[u32]` / +// `&[i32]` and the word-algebra ops over `&[u64]` — walks its input with +// `slice::as_chunks::()` (stable since 1.88): the main body iterates +// `&[[T; LANES]]` and feeds each chunk to `from_array` — a fixed-size load +// with no per-chunk bounds check and no `g * L` index arithmetic for LLVM to +// prove away — and the remainder is the EXACT tail slice. That tail is NOT +// peeled as a scalar loop: it is zero-padded into one register (`pad_tail`) +// and run through the SAME packed op as the body, with padding lanes never +// written back. (The strided byte-gather ops, the popcount-driven folds, and +// `blend_i32` are NOT in this family and say so in their own docs.) +// +// Measured reason (codegen witness, 2026-09-14): with an exact-length scalar +// tail LLVM fully unrolled it on aarch64 into 7 × (and, orr) on GPRs, which +// with the 2 index-mask ops read as 16 GPR logic ops in a facade op whose +// contract is "packed on every backend", while on AVX2 the same loop became +// `vpmaskmovq` masked vectors. Padding makes both arms the same shape: packed +// body, packed tail. What REMAINS is not zero: 4 GPR logic ops on v3 and 2 on +// aarch64, hand-classified as length/index arithmetic (`andl $7`, `& !63`), +// not lane data — the witness bounds their COUNT (`SLICE_GPR_CAP`), it does +// not classify them. What was measured is that count; no throughput +// comparison against the old peel has been made (the tail is a zero-init + +// two bounded `copy_from_slice`s + one packed op, and for inputs shorter than +// one register it IS the whole operation). +// +// `from_array` (not `from_slice`) because it exists on every backend's +// `U32x16`/`I32x16`/`U64x8` — the NEON and wasm `[..x4; 4]` fan-outs expose +// no `from_slice` — so the loops stay free of any `cfg(target_arch)`. + +/// Zero-pad a `< N`-element tail into one full register's worth of lanes so +/// the tail runs through the same packed op as the body. Padding lanes are +/// never written back: word-op callers copy out exactly `tail.len()` results, +/// predicate callers mask the bitmask down with [`tail_lane_bits`]. +#[inline(always)] +fn pad_tail(tail: &[T]) -> [T; N] { + let mut lanes = [T::default(); N]; + lanes[..tail.len()].copy_from_slice(tail); + lanes +} + +/// The low `n` bits set (`n < 16`): the lane-validity mask for a padded +/// 16-lane predicate tail, so a padding lane can never contribute a match. +#[inline(always)] +fn tail_lane_bits(n: usize) -> u16 { + debug_assert!(n < 16, "a tail is shorter than one register"); + ((1u32 << n) - 1) as u16 +} + +/// Packs `values[i] == needle` into `out_words`, one bit per element, +/// LSB-first within each `u64` word (bit `k` of word `w` corresponds to +/// element `w * 64 + k`). +/// +/// `out_words` is **fully overwritten**, not OR-ed into. Trailing bits in the +/// final word beyond `values.len()`, and any surplus words past +/// `ceil(len / 64)`, are written as `0`. +/// +/// Equality is exact bitwise comparison over the full `u32` range — `0` and +/// `u32::MAX` are ordinary needles, and there is no saturation, wrapping, or +/// signedness question to resolve. An empty `values` writes only zeros. +/// +/// Runs 16 lanes at a time through [`crate::simd::U32x16::eq_bitmask`]; the +/// final partial group is zero-padded into one register and run through the +/// same packed compare, with the padding lanes' bits masked off — no scalar +/// tail, so the tail cannot disagree with the body. +/// +/// # Panics +/// +/// Panics if `out_words.len() < values.len().div_ceil(64)`. +/// +/// # Examples +/// +/// ``` +/// use ndarray::simd::eq_u32_to_mask; +/// +/// let values = [7u32, 1, 7, 2]; +/// let mut words = [0u64; 1]; +/// eq_u32_to_mask(&values, 7, &mut words); +/// // elements 0 and 2 match → bits 0 and 2 → 0b0101 +/// assert_eq!(words[0], 0b0101); +/// ``` +#[inline] +pub fn eq_u32_to_mask(values: &[u32], needle: u32, out_words: &mut [u64]) { + let n = values.len(); + let words = mask_words_for(n); + assert!(out_words.len() >= words, "eq_u32_to_mask: out_words.len()={} < required {}", out_words.len(), words); + + // Zero first: makes the "trailing bits are 0" guarantee structural. + for w in out_words.iter_mut() { + *w = 0; + } + + let needle_v = crate::simd::U32x16::splat(needle); + let (chunks, tail) = values.as_chunks::<16>(); + for (g, chunk) in chunks.iter().enumerate() { + let bits = crate::simd::U32x16::from_array(*chunk).eq_bitmask(needle_v); + out_words[g / 4] |= (bits as u64) << ((g % 4) * 16); + } + if !tail.is_empty() { + let g = chunks.len(); + let bits = crate::simd::U32x16::from_array(pad_tail(tail)).eq_bitmask(needle_v) & tail_lane_bits(tail.len()); + out_words[g / 4] |= (bits as u64) << ((g % 4) * 16); + } +} + +/// Packs `read_le_u32(bytes, first_offset + i * stride_bytes) == needle` into +/// `out_words`, one bit per element, LSB-first within each `u64` word — the +/// **strided** sibling of [`eq_u32_to_mask`], for scanning one `u32` field of +/// an AoS/facet row layout (e.g. a 4-byte classid at a fixed offset inside a +/// 512-byte row) without gathering the column into a contiguous copy first. +/// +/// Element `i` is the little-endian `u32` at byte offset +/// `first_offset + i * stride_bytes`. `stride_bytes == 4` reads a contiguous +/// `u32` column and takes a dedicated contiguous path (one 64-byte window per +/// 16 elements — the same load shape as [`eq_u32_to_mask`]); `stride_bytes +/// == 0` re-reads the same field `count` times, which is legal and produces +/// an all-ones or all-zeros mask. +/// +/// `out_words` is **fully overwritten**, not OR-ed into; trailing bits and +/// surplus words are written `0`, exactly as in [`eq_u32_to_mask`]. +/// +/// The field loads are scalar by construction — at row strides ≥ one cache +/// line each element lives on its own line, so the walk is memory-bound and +/// a hardware gather buys nothing; SIMD earns its keep in the 16-wide +/// compare ([`crate::simd::U32x16::eq_bitmask`]) exactly as the contiguous +/// primitive does. Loads are `u32::from_le_bytes` over byte slices, so no +/// alignment is required of `bytes`. +/// +/// # Panics +/// +/// Panics if `out_words.len() < count.div_ceil(64)`, or if any element's four +/// bytes would fall outside `bytes` (checked up front, including overflow of +/// the offset arithmetic — the loop never reads out of bounds). +/// +/// # Examples +/// +/// ``` +/// use ndarray::simd::eq_u32_strided_to_mask; +/// +/// // Three 16-byte "facets"; the classid is the leading u32 of each. +/// let mut rows = vec![0u8; 48]; +/// rows[0..4].copy_from_slice(&7u32.to_le_bytes()); +/// rows[16..20].copy_from_slice(&9u32.to_le_bytes()); +/// rows[32..36].copy_from_slice(&7u32.to_le_bytes()); +/// let mut words = [0u64; 1]; +/// eq_u32_strided_to_mask(&rows, 0, 16, 3, 7, &mut words); +/// assert_eq!(words[0], 0b101); +/// ``` +#[inline] +pub fn eq_u32_strided_to_mask( + bytes: &[u8], first_offset: usize, stride_bytes: usize, count: usize, needle: u32, out_words: &mut [u64], +) { + let words = mask_words_for(count); + assert!( + out_words.len() >= words, + "eq_u32_strided_to_mask: out_words.len()={} < required {}", + out_words.len(), + words + ); + if count > 0 { + // Bounds of the LAST element, computed with overflow checks so a + // pathological stride cannot wrap around into a bogus in-bounds read. + let last_start = (count - 1) + .checked_mul(stride_bytes) + .and_then(|o| o.checked_add(first_offset)) + .expect("eq_u32_strided_to_mask: offset arithmetic overflow"); + let last_end = last_start + .checked_add(4) + .expect("eq_u32_strided_to_mask: offset arithmetic overflow"); + assert!( + last_end <= bytes.len(), + "eq_u32_strided_to_mask: element {} at byte {}..{} is out of bounds (len {})", + count - 1, + last_start, + last_end, + bytes.len() + ); + } + + for w in out_words.iter_mut() { + *w = 0; + } + + #[inline(always)] + fn read_le_u32(bytes: &[u8], off: usize) -> u32 { + u32::from_le_bytes([bytes[off], bytes[off + 1], bytes[off + 2], bytes[off + 3]]) + } + + let needle_v = crate::simd::U32x16::splat(needle); + let groups = count / 16; + if stride_bytes == 4 { + // Contiguous lane (a facet-major column): 16 elements are ONE 64-byte + // window, so the full groups are exactly the `as_chunks::<64>()` of + // the byte range they cover — fixed-size windows with no per-element + // bounds check (bounds were proven above for the last element, so the + // sub-slice cannot panic). The `[u32; 16]` built from each window is + // the same register-sized temporary the general path uses; what this + // removes is the checks, not the temporary. + let (windows, _) = bytes[first_offset..first_offset + groups * 64].as_chunks::<64>(); + for (g, window) in windows.iter().enumerate() { + let lanes: [u32; 16] = core::array::from_fn(|k| { + u32::from_le_bytes([window[4 * k], window[4 * k + 1], window[4 * k + 2], window[4 * k + 3]]) + }); + let bits = crate::simd::U32x16::from_array(lanes).eq_bitmask(needle_v); + out_words[g / 4] |= (bits as u64) << ((g % 4) * 16); + } + } else { + for g in 0..groups { + let base = first_offset + g * 16 * stride_bytes; + let lanes: [u32; 16] = core::array::from_fn(|k| read_le_u32(bytes, base + k * stride_bytes)); + let bits = crate::simd::U32x16::from_array(lanes).eq_bitmask(needle_v); + out_words[g / 4] |= (bits as u64) << ((g % 4) * 16); + } + } + for i in (groups * 16)..count { + if read_le_u32(bytes, first_offset + i * stride_bytes) == needle { + out_words[i / 64] |= 1u64 << (i % 64); + } + } +} + +/// Packs `values[i] > threshold` (**signed** comparison) into `out_words`, +/// one bit per element, LSB-first within each `u64` word (bit `k` of word `w` +/// corresponds to element `w * 64 + k`). +/// +/// `out_words` is **fully overwritten**, not OR-ed into. Trailing bits in the +/// final word beyond `values.len()`, and any surplus words past +/// `ceil(len / 64)`, are written as `0`. +/// +/// Comparison is two's-complement signed and strict (`>`, never `>=`); it is +/// exact with no saturation or wrapping: +/// * `threshold == i32::MIN` sets every lane except those equal to `i32::MIN`. +/// * `threshold == i32::MAX` sets nothing — no `i32` exceeds it. +/// * Negative values compare as signed, *not* as bit patterns: `-1 > 0` is +/// `false` even though the same bits compare greater unsigned. +/// +/// An empty `values` writes only zeros. +/// +/// Runs 16 lanes at a time through [`crate::simd::I32x16::gt_bitmask`]; the +/// final partial group is zero-padded into one register and run through the +/// same packed compare, with the padding lanes' bits masked off. +/// +/// # Panics +/// +/// Panics if `out_words.len() < values.len().div_ceil(64)`. +/// +/// # Examples +/// +/// ``` +/// use ndarray::simd::gt_i32_to_mask; +/// +/// let values = [5i32, -5, 0, i32::MAX]; +/// let mut words = [0u64; 1]; +/// gt_i32_to_mask(&values, 0, &mut words); +/// // elements 0 and 3 exceed 0 → bits 0 and 3 → 0b1001 +/// assert_eq!(words[0], 0b1001); +/// ``` +#[inline] +pub fn gt_i32_to_mask(values: &[i32], threshold: i32, out_words: &mut [u64]) { + let n = values.len(); + let words = mask_words_for(n); + assert!(out_words.len() >= words, "gt_i32_to_mask: out_words.len()={} < required {}", out_words.len(), words); + + for w in out_words.iter_mut() { + *w = 0; + } + + let threshold_v = crate::simd::I32x16::splat(threshold); + let (chunks, tail) = values.as_chunks::<16>(); + for (g, chunk) in chunks.iter().enumerate() { + let bits = crate::simd::I32x16::from_array(*chunk).gt_bitmask(threshold_v); + out_words[g / 4] |= (bits as u64) << ((g % 4) * 16); + } + if !tail.is_empty() { + let g = chunks.len(); + let bits = crate::simd::I32x16::from_array(pad_tail(tail)).gt_bitmask(threshold_v) & tail_lane_bits(tail.len()); + out_words[g / 4] |= (bits as u64) << ((g % 4) * 16); + } +} + +/// `dst = a & b`, elementwise over `u64` mask words. +/// +/// Pure bitwise AND — no element-count awareness, so the caller's bit-order +/// convention (element `i` at bit `i % 64` of word `i / 64`) is preserved +/// automatically, including the trailing-zero guarantee: zero AND anything is +/// zero, so a conforming pair of inputs yields a conforming output. +/// +/// `dst` must **not** overlap `a` or `b`; use [`mask_and_assign`] for the +/// in-place case (Rust's borrow rules already prevent the overlap in safe +/// code, so this is a note about which function to reach for, not a hazard). +/// +/// # Panics +/// +/// Panics unless `a.len() == b.len() == dst.len()`. +#[inline] +pub fn mask_and(a: &[u64], b: &[u64], dst: &mut [u64]) { + assert_eq!(a.len(), b.len(), "mask_and: a/b length mismatch"); + assert_eq!(a.len(), dst.len(), "mask_and: a/dst length mismatch"); + const L: usize = crate::simd::U64x8::LANES; + let (ca, ta) = a.as_chunks::(); + let (cb, tb) = b.as_chunks::(); + let (cd, td) = dst.as_chunks_mut::(); + for ((x, y), d) in ca.iter().zip(cb).zip(cd.iter_mut()) { + let va = crate::simd::U64x8::from_array(*x); + let vb = crate::simd::U64x8::from_array(*y); + *d = (va & vb).to_array(); + } + if !ta.is_empty() { + let va = crate::simd::U64x8::from_array(pad_tail(ta)); + let vb = crate::simd::U64x8::from_array(pad_tail(tb)); + td.copy_from_slice(&(va & vb).to_array()[..td.len()]); + } +} + +/// `dst = a | b`, elementwise over `u64` mask words. +/// +/// Pure bitwise OR. Note the trailing-zero asymmetry versus [`mask_and`]: OR +/// preserves the guarantee only if **both** inputs already conform, because a +/// stray high bit in either operand survives. Every mask this module produces +/// conforms, so composing them is safe; a hand-built mask word is the caller's +/// responsibility. +/// +/// `dst` must not overlap `a` or `b`; use [`mask_or_assign`] in-place. +/// +/// # Panics +/// +/// Panics unless `a.len() == b.len() == dst.len()`. +#[inline] +pub fn mask_or(a: &[u64], b: &[u64], dst: &mut [u64]) { + assert_eq!(a.len(), b.len(), "mask_or: a/b length mismatch"); + assert_eq!(a.len(), dst.len(), "mask_or: a/dst length mismatch"); + const L: usize = crate::simd::U64x8::LANES; + let (ca, ta) = a.as_chunks::(); + let (cb, tb) = b.as_chunks::(); + let (cd, td) = dst.as_chunks_mut::(); + for ((x, y), d) in ca.iter().zip(cb).zip(cd.iter_mut()) { + let va = crate::simd::U64x8::from_array(*x); + let vb = crate::simd::U64x8::from_array(*y); + *d = (va | vb).to_array(); + } + if !ta.is_empty() { + let va = crate::simd::U64x8::from_array(pad_tail(ta)); + let vb = crate::simd::U64x8::from_array(pad_tail(tb)); + td.copy_from_slice(&(va | vb).to_array()[..td.len()]); + } +} + +/// `dst &= src`, elementwise over `u64` mask words. +/// +/// The in-place form of [`mask_and`] — this is what a fused predicate plan +/// uses to narrow an accumulator, and what an ABI-level `mask_and(a, b, dst)` +/// with `dst` aliasing an operand must route to. +/// +/// # Panics +/// +/// Panics if `dst.len() != src.len()`. +#[inline] +pub fn mask_and_assign(dst: &mut [u64], src: &[u64]) { + assert_eq!(dst.len(), src.len(), "mask_and_assign: length mismatch"); + const L: usize = crate::simd::U64x8::LANES; + let (cd, td) = dst.as_chunks_mut::(); + let (cs, ts) = src.as_chunks::(); + for (d, s) in cd.iter_mut().zip(cs) { + let vd = crate::simd::U64x8::from_array(*d); + let vs = crate::simd::U64x8::from_array(*s); + *d = (vd & vs).to_array(); + } + if !td.is_empty() { + let vd = crate::simd::U64x8::from_array(pad_tail(td)); + let vs = crate::simd::U64x8::from_array(pad_tail(ts)); + td.copy_from_slice(&(vd & vs).to_array()[..td.len()]); + } +} + +/// `dst |= src`, elementwise over `u64` mask words. +/// +/// The in-place form of [`mask_or`]. Same trailing-zero caveat as `mask_or`: +/// OR only preserves the convention if `src` conforms to it. +/// +/// # Panics +/// +/// Panics if `dst.len() != src.len()`. +#[inline] +pub fn mask_or_assign(dst: &mut [u64], src: &[u64]) { + assert_eq!(dst.len(), src.len(), "mask_or_assign: length mismatch"); + const L: usize = crate::simd::U64x8::LANES; + let (cd, td) = dst.as_chunks_mut::(); + let (cs, ts) = src.as_chunks::(); + for (d, s) in cd.iter_mut().zip(cs) { + let vd = crate::simd::U64x8::from_array(*d); + let vs = crate::simd::U64x8::from_array(*s); + *d = (vd | vs).to_array(); + } + if !td.is_empty() { + let vd = crate::simd::U64x8::from_array(pad_tail(td)); + let vs = crate::simd::U64x8::from_array(pad_tail(ts)); + td.copy_from_slice(&(vd | vs).to_array()[..td.len()]); + } +} + +/// `dst = a & !b`, elementwise over `u64` mask words — "a minus b" as a +/// bitmask set difference (every bit set in `a` but not in `b`). +/// +/// # Tail-bit semantics +/// +/// `!b` sets every bit of `b`'s tail — the padding bits past whatever +/// logical row count `b` represents — because bitwise NOT has no notion of +/// "past the end" and will happily flip a conforming (zero) tail to all +/// ones. That looks like the same hazard [`mask_or`] warns about, but the +/// AND with `a` recovers it: `a & !b` is a bitwise subset of `a` (every bit +/// set in the result is also set in `a`), so **`dst`'s tail is zero +/// whenever `a`'s tail is zero, regardless of what `!b`'s tail does.** This +/// is the same pre-conforming-inputs contract `mask_or` documents — a +/// caller holding a possibly-non-conforming `a` must clear `a`'s tail +/// itself (the lgj-abi kernel does, against its own known `n_rows`); a +/// conforming `a` composes safely against any `b`, tail included. +/// +/// `dst` must not overlap `a` or `b`; use [`mask_andnot_assign`] for the +/// in-place case (Rust's borrow rules already prevent the overlap in safe +/// code, so this is a note about which function to reach for, not a +/// hazard). +/// +/// # Panics +/// +/// Panics unless `a.len() == b.len() == dst.len()`. +#[inline] +pub fn mask_andnot(a: &[u64], b: &[u64], dst: &mut [u64]) { + assert_eq!(a.len(), b.len(), "mask_andnot: a/b length mismatch"); + assert_eq!(a.len(), dst.len(), "mask_andnot: a/dst length mismatch"); + const L: usize = crate::simd::U64x8::LANES; + let (ca, ta) = a.as_chunks::(); + let (cb, tb) = b.as_chunks::(); + let (cd, td) = dst.as_chunks_mut::(); + for ((x, y), d) in ca.iter().zip(cb).zip(cd.iter_mut()) { + let va = crate::simd::U64x8::from_array(*x); + let vb = crate::simd::U64x8::from_array(*y); + *d = (va & !vb).to_array(); + } + if !ta.is_empty() { + let va = crate::simd::U64x8::from_array(pad_tail(ta)); + let vb = crate::simd::U64x8::from_array(pad_tail(tb)); + td.copy_from_slice(&(va & !vb).to_array()[..td.len()]); + } +} + +/// `a &= !b`, elementwise over `u64` mask words. +/// +/// The in-place form of [`mask_andnot`] — same tail-bit contract: the +/// result is a bitwise subset of the (pre-update) `a`, so `a`'s tail stays +/// zero whenever it started zero, regardless of what `b`'s tail holds. +/// +/// # Panics +/// +/// Panics if `a.len() != b.len()`. +#[inline] +pub fn mask_andnot_assign(a: &mut [u64], b: &[u64]) { + assert_eq!(a.len(), b.len(), "mask_andnot_assign: length mismatch"); + const L: usize = crate::simd::U64x8::LANES; + let (ca, ta) = a.as_chunks_mut::(); + let (cb, tb) = b.as_chunks::(); + for (x, y) in ca.iter_mut().zip(cb) { + let va = crate::simd::U64x8::from_array(*x); + let vb = crate::simd::U64x8::from_array(*y); + *x = (va & !vb).to_array(); + } + if !ta.is_empty() { + let va = crate::simd::U64x8::from_array(pad_tail(ta)); + let vb = crate::simd::U64x8::from_array(pad_tail(tb)); + ta.copy_from_slice(&(va & !vb).to_array()[..ta.len()]); + } +} + +/// `dst = ternlog::(a, b, c)`, elementwise over `u64` mask words — any +/// 3-input Boolean function of three masks in one pass. +/// +/// `IMM` is the 8-bit truth table in Intel's VPTERNLOG convention (index +/// `(a<<2)|(b<<1)|c`, result bit `(IMM >> index) & 1`); the named tables in +/// [`crate::simd::ternlog`] (`AND3`, `OR3`, `MAJ3`, `AND2_ANDNOT`, …) are +/// the sanctioned spellings. This is the mask-op family's general member: +/// [`mask_and`] is `mask_ternlog::<{ ternlog::AND2 }>` with `c` ignored, +/// [`mask_andnot`] is `AND2_ANDNOT` with `c` ignored, and the composed +/// `a & b & c` that a consumer would otherwise spell as two `mask_and_assign` +/// passes through a scratch buffer is ONE `AND3` pass here — one +/// `VPTERNLOGQ` per 512 bits on AVX-512, the polyfill elsewhere. +/// +/// # Tail-bit semantics +/// +/// Whether `dst`'s tail conforms depends on the truth table, not on the +/// inputs alone: the tail of every conforming input is zero, so `dst`'s tail +/// is `IMM & 1` replicated — **zero iff `IMM` is even** (index 0 = all-zero +/// inputs maps to 0). Every named table in [`crate::simd::ternlog`] is even. +/// An odd `IMM` (one whose function is true of `(0,0,0)`) sets every tail bit +/// and the caller must clear the tail against its own known row count, exactly +/// as [`mask_or`] documents for a non-conforming operand. For the +/// subset-shaped tables (`AND3`, `AND2_ANDNOT`, `AND_ANDNOT2`, `AND2`) the +/// stronger [`mask_andnot`] guarantee also holds: the result is a bitwise +/// subset of `a`, so `dst`'s tail is zero whenever `a`'s is, regardless of +/// `b` and `c`. +/// +/// `dst` must not overlap `a`, `b` or `c`; use [`mask_ternlog_assign`] for +/// the in-place case. +/// +/// # Panics +/// +/// Panics unless `a.len() == b.len() == c.len() == dst.len()`. +#[inline] +pub fn mask_ternlog(a: &[u64], b: &[u64], c: &[u64], dst: &mut [u64]) { + assert_eq!(a.len(), b.len(), "mask_ternlog: a/b length mismatch"); + assert_eq!(a.len(), c.len(), "mask_ternlog: a/c length mismatch"); + assert_eq!(a.len(), dst.len(), "mask_ternlog: a/dst length mismatch"); + const L: usize = crate::simd::U64x8::LANES; + let (ca, ta) = a.as_chunks::(); + let (cb, tb) = b.as_chunks::(); + let (cc, tc) = c.as_chunks::(); + let (cd, td) = dst.as_chunks_mut::(); + for (((x, y), z), d) in ca.iter().zip(cb).zip(cc).zip(cd.iter_mut()) { + let va = crate::simd::U64x8::from_array(*x); + let vb = crate::simd::U64x8::from_array(*y); + let vc = crate::simd::U64x8::from_array(*z); + *d = va.ternlog::(vb, vc).to_array(); + } + if !ta.is_empty() { + let va = crate::simd::U64x8::from_array(pad_tail(ta)); + let vb = crate::simd::U64x8::from_array(pad_tail(tb)); + let vc = crate::simd::U64x8::from_array(pad_tail(tc)); + td.copy_from_slice(&va.ternlog::(vb, vc).to_array()[..td.len()]); + } +} + +/// `a = ternlog::(a, b, c)`, elementwise over `u64` mask words. +/// +/// The in-place form of [`mask_ternlog`] — `a` is the first truth-table +/// operand AND the destination, which is the shape a fused predicate plan +/// wants when narrowing an accumulator against two more masks in one pass +/// (`selected = selected & src & gate` as `AND3`). Same tail contract as +/// [`mask_ternlog`]. +/// +/// # Panics +/// +/// Panics unless `a.len() == b.len() == c.len()`. +#[inline] +pub fn mask_ternlog_assign(a: &mut [u64], b: &[u64], c: &[u64]) { + assert_eq!(a.len(), b.len(), "mask_ternlog_assign: a/b length mismatch"); + assert_eq!(a.len(), c.len(), "mask_ternlog_assign: a/c length mismatch"); + const L: usize = crate::simd::U64x8::LANES; + let (ca, ta) = a.as_chunks_mut::(); + let (cb, tb) = b.as_chunks::(); + let (cc, tc) = c.as_chunks::(); + for ((x, y), z) in ca.iter_mut().zip(cb).zip(cc) { + let va = crate::simd::U64x8::from_array(*x); + let vb = crate::simd::U64x8::from_array(*y); + let vc = crate::simd::U64x8::from_array(*z); + *x = va.ternlog::(vb, vc).to_array(); + } + if !ta.is_empty() { + let va = crate::simd::U64x8::from_array(pad_tail(ta)); + let vb = crate::simd::U64x8::from_array(pad_tail(tb)); + let vc = crate::simd::U64x8::from_array(pad_tail(tc)); + ta.copy_from_slice(&va.ternlog::(vb, vc).to_array()[..ta.len()]); + } +} + +/// Sum of `values[i]` where mask bit `i` is set, widened to `i64`. +/// +/// Bit order is the module convention: element `i` is bit `i % 64` of +/// `mask_words[i / 64]`. +/// +/// ## Overflow behaviour (precise) +/// +/// Each element is widened to `i64` **before** accumulation, so no +/// intermediate can overflow at any realistic length: the worst case is +/// `n × |i32::MIN|`, which stays inside `i64` for every `n < 2^32` — i.e. for +/// every slice that can exist in a 64-bit address space at 4 bytes per +/// element. The accumulation is nevertheless written as `wrapping_add` so +/// that the theoretical `n ≥ 2^32` case has defined behaviour (two's-complement +/// wrap) rather than a debug-only panic that a release build would silently +/// disagree with. An empty mask, or a mask with no bits set, returns `0`. +/// +/// **Mask bits at or beyond `values.len()` are ignored**, not summed and not +/// an error: the final word is masked down to the valid element count before +/// its bits are walked. This makes the function total for any conforming or +/// over-long mask, and means a caller cannot read past the value lane by +/// handing over a dirty tail. +/// +/// ## Why this one is not a 16-lane reduce +/// +/// The obvious vector shape — load `I32x16`, zero the unselected lanes, +/// `reduce_sum()` — is **wrong**, and quietly so: `reduce_sum` on `I32x16` +/// accumulates in `i32`, and 16 lanes near `i32::MAX` overflow it while the +/// widened contract promises they cannot. Preserving the `i64` guarantee is +/// worth more than the lanes here, so the body walks set bits with +/// `u64::trailing_zeros` (one `TZCNT`/`RBIT+CLZ` per selected element, and +/// entire zero words skipped in one test). Cost is proportional to the +/// popcount, not the row count, which is the right shape for a selective +/// filter anyway. +/// +/// # Panics +/// +/// Panics if `mask_words.len() < values.len().div_ceil(64)`. +/// +/// # Examples +/// +/// ``` +/// use ndarray::simd::masked_sum_i32; +/// +/// let values = [10i32, 20, 30, 40]; +/// // bits 0 and 2 set → 10 + 30 +/// assert_eq!(masked_sum_i32(&values, &[0b0101]), 40); +/// ``` +#[inline] +pub fn masked_sum_i32(values: &[i32], mask_words: &[u64]) -> i64 { + let n = values.len(); + let words = mask_words_for(n); + assert!( + mask_words.len() >= words, + "masked_sum_i32: mask_words.len()={} < required {}", + mask_words.len(), + words + ); + + let mut acc: i64 = 0; + for (w, &word) in mask_words.iter().take(words).enumerate() { + let base = w * 64; + let mut bits = word; + // Clamp the final partial word to the valid element count so a dirty + // tail can never index past `values`. + let valid = n - base; + if valid < 64 { + bits &= (1u64 << valid) - 1; + } + while bits != 0 { + let lane = bits.trailing_zeros() as usize; + acc = acc.wrapping_add(values[base + lane] as i64); + bits &= bits - 1; + } + } + acc +} + +/// Sum a sub-word group field out of a **strided** record, over the records a +/// mask selects, widened to `i128` and range-checked into `i64`. +/// +/// The shape this exists for: a row-strided store whose each record carries a small +/// content-blind register, read under a runtime grouping — `groups × group_bytes` +/// little-endian fields per record. `lance-graph-java`'s V3 facet is the +/// motivating case (512-byte rows, a 12-byte register read as `6×2` / `4×3` / +/// `3×4`), but nothing here is specific to it. +/// +/// # Why this lives HERE +/// +/// It is the primitive a consumer would otherwise hand-roll with raw intrinsics, +/// which is exactly what the "all SIMD from `ndarray::simd`" invariant exists to +/// prevent. [`masked_sum_i32`] is contiguous `i32`; +/// [`eq_u32_strided_to_mask`] reads one aligned `u32` per record. Neither covers +/// "gather a sub-word group out of a strided register and widen-accumulate", so +/// the consumer had a real gap and this closes it. +/// +/// # Vectorisation, honestly +/// +/// **This kernel is scalar, and measurement is why — not oversight.** The access +/// pattern is one small register per record at a large stride (512 bytes in the +/// motivating case), so every record is on its own cache line and the loop is +/// memory-bound. The per-record work is 12 bytes; a vector register is 32-64. +/// There is no way to vector-load several records' registers at once because +/// they are not adjacent, and widening 6 `u16`s within one record does not fill +/// a lane. Vectorising the *decode* would optimise the part that is already +/// free. +/// +/// Should a caller ever present a CONTIGUOUS or small-stride variant, that is a +/// different primitive with a different name, and it would genuinely vectorise — +/// this one should not grow a flag for it. +/// +/// # Overflow +/// +/// Accumulates in `i128` and range-checks once, returning `None` rather than a +/// wrapped value. `i64` is not closed under this reduction: with +/// `group_bytes = 4` a single record contributes up to `groups × (2³² − 1)`. +/// +/// # Panics +/// +/// If `group_bytes` is not in `1..=4`, if `mask_words` is too short for +/// `n_records`, or if the last selected record's field would read past `bytes`. +/// Each is a caller contract violation rather than a recoverable condition. +/// +/// ``` +/// use ndarray::simd::masked_strided_group_sum; +/// +/// // Two 8-byte records; the register starts at byte 2 and holds 3 × u16 LE. +/// let mut b = vec![0u8; 16]; +/// b[2..8].copy_from_slice(&[1, 0, 2, 0, 3, 0]); // record 0 -> 1 + 2 + 3 +/// b[10..16].copy_from_slice(&[10, 0, 20, 0, 30, 0]); // record 1 -> 60 +/// // mask selects record 0 only +/// assert_eq!(masked_strided_group_sum(&b, 2, 8, 2, 3, 2, &[0b01]), Some(6)); +/// // both records +/// assert_eq!(masked_strided_group_sum(&b, 2, 8, 2, 3, 2, &[0b11]), Some(66)); +/// ``` +#[inline] +pub fn masked_strided_group_sum( + bytes: &[u8], first_offset: usize, stride_bytes: usize, n_records: usize, groups: usize, group_bytes: usize, + mask_words: &[u64], +) -> Option { + assert!((1..=4).contains(&group_bytes), "masked_strided_group_sum: group_bytes={group_bytes} outside 1..=4"); + let words = mask_words_for(n_records); + assert!( + mask_words.len() >= words, + "masked_strided_group_sum: mask_words.len()={} < required {}", + mask_words.len(), + words + ); + + let mut acc: i128 = 0; + for (w, &word) in mask_words.iter().take(words).enumerate() { + let base = w * 64; + if base >= n_records { + break; + } + let mut bits = word; + // Clamp the final partial word so a dirty tail cannot address a record + // that does not exist. Same guard, same reason, as `masked_sum_i32`. + let valid = n_records - base; + if valid < 64 { + bits &= (1u64 << valid) - 1; + } + while bits != 0 { + let rec = base + bits.trailing_zeros() as usize; + bits &= bits - 1; + let reg = rec * stride_bytes + first_offset; + let end = reg + groups * group_bytes; + assert!( + end <= bytes.len(), + "masked_strided_group_sum: record {rec} reads {reg}..{end}, past len {}", + bytes.len() + ); + for g in 0..groups { + let o = reg + g * group_bytes; + // Byte-wise, not a widened load: `o` is not guaranteed aligned + // for a 3-byte grouping, and an unaligned wide read is UB in + // Rust even where the hardware tolerates it. + let mut v: u32 = 0; + for k in 0..group_bytes { + v |= (bytes[o + k] as u32) << (8 * k); + } + acc += v as i128; + } + } + } + i64::try_from(acc).ok() +} + +// ──────────────────────────────────────────────────────────────────────── +// The closed comparison family + mask complement/xor/any + care-masked +// register match + masked min/max + blend (the DuckDB-vector-execution set, +// added 2026-09-13 for `lance-graph-duckmask` and lgj-abi D-MRL-1a). +// +// Same normative bit order and trailing-zero guarantee as everything above. +// Every writer below is a full overwrite. Ordered compares are SIGNED for the +// `i32` family; equality/inequality is exact bitwise. +// +// Why `!gt`-style derivations rather than threshold shifting: `x <= t` as +// `!(x > t)` is exact at every boundary including `i32::MIN`/`i32::MAX`, +// whereas `x < t` as `x > t - 1` underflows at `t == i32::MIN`. The tail of +// a complemented mask is re-cleared against the known element count, so the +// guarantee stays structural. +// ──────────────────────────────────────────────────────────────────────── + +/// Clear every bit at or past element `n` — the tail of the last live word +/// and every surplus word. Shared by every complementing writer below. +#[inline(always)] +fn clear_mask_tail(out_words: &mut [u64], n: usize) { + let words = mask_words_for(n); + if words > 0 && !n.is_multiple_of(64) { + out_words[words - 1] &= (1u64 << (n % 64)) - 1; + } + for w in out_words.iter_mut().skip(words) { + *w = 0; + } +} + +/// Packs `values[i] < threshold` (signed) into `out_words`; full overwrite, +/// trailing bits zero. Lowered as `threshold > values[i]` through +/// [`crate::simd::I32x16::gt_bitmask`], so it is exact at `i32::MIN` (never +/// `x > t - 1`). +/// +/// # Panics +/// +/// Panics if `out_words.len() < values.len().div_ceil(64)`. +/// +/// # Examples +/// +/// ``` +/// use ndarray::simd::lt_i32_to_mask; +/// +/// let values = [5i32, -5, 0, 10]; +/// let mut words = [0u64; 1]; +/// lt_i32_to_mask(&values, 0, &mut words); +/// // only -5 is less than 0 → bit 1 → 0b0010 +/// assert_eq!(words[0], 0b0010); +/// ``` +#[inline] +pub fn lt_i32_to_mask(values: &[i32], threshold: i32, out_words: &mut [u64]) { + let n = values.len(); + let words = mask_words_for(n); + assert!(out_words.len() >= words, "lt_i32_to_mask: out_words.len()={} < required {}", out_words.len(), words); + for w in out_words.iter_mut() { + *w = 0; + } + let t = crate::simd::I32x16::splat(threshold); + let (chunks, tail) = values.as_chunks::<16>(); + for (g, chunk) in chunks.iter().enumerate() { + let bits = t.gt_bitmask(crate::simd::I32x16::from_array(*chunk)); + out_words[g / 4] |= (bits as u64) << ((g % 4) * 16); + } + if !tail.is_empty() { + let g = chunks.len(); + let bits = t.gt_bitmask(crate::simd::I32x16::from_array(pad_tail(tail))) & tail_lane_bits(tail.len()); + out_words[g / 4] |= (bits as u64) << ((g % 4) * 16); + } +} + +/// Packs `values[i] >= threshold` (signed): the complement of +/// [`lt_i32_to_mask`] with the tail re-cleared. Full overwrite. +/// +/// # Panics +/// +/// Panics if `out_words.len() < values.len().div_ceil(64)`. +/// +/// # Examples +/// +/// ``` +/// use ndarray::simd::ge_i32_to_mask; +/// +/// let values = [5i32, -5, 0, 10]; +/// let mut words = [0u64; 1]; +/// ge_i32_to_mask(&values, 0, &mut words); +/// // 5, 0 and 10 are >= 0 → bits 0, 2, 3 → 0b1101 +/// assert_eq!(words[0], 0b1101); +/// ``` +#[inline] +pub fn ge_i32_to_mask(values: &[i32], threshold: i32, out_words: &mut [u64]) { + lt_i32_to_mask(values, threshold, out_words); + for w in out_words.iter_mut() { + *w = !*w; + } + clear_mask_tail(out_words, values.len()); +} + +/// Packs `values[i] <= threshold` (signed): the complement of +/// [`gt_i32_to_mask`] with the tail re-cleared. Full overwrite. +/// +/// # Panics +/// +/// Panics if `out_words.len() < values.len().div_ceil(64)`. +/// +/// # Examples +/// +/// ``` +/// use ndarray::simd::le_i32_to_mask; +/// +/// let values = [5i32, -5, 0, 10]; +/// let mut words = [0u64; 1]; +/// le_i32_to_mask(&values, 0, &mut words); +/// // -5 and 0 are <= 0 → bits 1, 2 → 0b0110 +/// assert_eq!(words[0], 0b0110); +/// ``` +#[inline] +pub fn le_i32_to_mask(values: &[i32], threshold: i32, out_words: &mut [u64]) { + gt_i32_to_mask(values, threshold, out_words); + for w in out_words.iter_mut() { + *w = !*w; + } + clear_mask_tail(out_words, values.len()); +} + +/// Packs `values[i] != needle` into `out_words` in ONE pass as +/// `(needle > v) | (v > needle)` — no complement, so the tail is zero by +/// construction. Full overwrite. +/// +/// # Panics +/// +/// Panics if `out_words.len() < values.len().div_ceil(64)`. +/// +/// # Examples +/// +/// ``` +/// use ndarray::simd::ne_i32_to_mask; +/// +/// let values = [7i32, 1, 7, 2]; +/// let mut words = [0u64; 1]; +/// ne_i32_to_mask(&values, 7, &mut words); +/// // elements 1 and 3 differ from 7 → bits 1 and 3 → 0b1010 +/// assert_eq!(words[0], 0b1010); +/// ``` +#[inline] +pub fn ne_i32_to_mask(values: &[i32], needle: i32, out_words: &mut [u64]) { + let n = values.len(); + let words = mask_words_for(n); + assert!(out_words.len() >= words, "ne_i32_to_mask: out_words.len()={} < required {}", out_words.len(), words); + for w in out_words.iter_mut() { + *w = 0; + } + let t = crate::simd::I32x16::splat(needle); + let (chunks, tail) = values.as_chunks::<16>(); + for (g, chunk) in chunks.iter().enumerate() { + let v = crate::simd::I32x16::from_array(*chunk); + let bits = t.gt_bitmask(v) | v.gt_bitmask(t); + out_words[g / 4] |= (bits as u64) << ((g % 4) * 16); + } + if !tail.is_empty() { + let g = chunks.len(); + let v = crate::simd::I32x16::from_array(pad_tail(tail)); + let bits = (t.gt_bitmask(v) | v.gt_bitmask(t)) & tail_lane_bits(tail.len()); + out_words[g / 4] |= (bits as u64) << ((g % 4) * 16); + } +} + +/// Packs `values[i] == needle` (signed lanes, exact): the complement of +/// [`ne_i32_to_mask`] with the tail re-cleared. Full overwrite. The unsigned +/// sibling is [`eq_u32_to_mask`]; the two agree on every bit pattern. +/// +/// # Panics +/// +/// Panics if `out_words.len() < values.len().div_ceil(64)`. +/// +/// # Examples +/// +/// ``` +/// use ndarray::simd::eq_i32_to_mask; +/// +/// let values = [7i32, 1, 7, 2]; +/// let mut words = [0u64; 1]; +/// eq_i32_to_mask(&values, 7, &mut words); +/// // elements 0 and 2 equal 7 → bits 0 and 2 → 0b0101 +/// assert_eq!(words[0], 0b0101); +/// ``` +#[inline] +pub fn eq_i32_to_mask(values: &[i32], needle: i32, out_words: &mut [u64]) { + ne_i32_to_mask(values, needle, out_words); + for w in out_words.iter_mut() { + *w = !*w; + } + clear_mask_tail(out_words, values.len()); +} + +/// Packs `values[i] != needle`: the complement of [`eq_u32_to_mask`] with the +/// tail re-cleared. Full overwrite. +/// +/// # Panics +/// +/// Panics if `out_words.len() < values.len().div_ceil(64)`. +/// +/// # Examples +/// +/// ``` +/// use ndarray::simd::ne_u32_to_mask; +/// +/// let values = [7u32, 1, 7, 2]; +/// let mut words = [0u64; 1]; +/// ne_u32_to_mask(&values, 7, &mut words); +/// // elements 1 and 3 differ from 7 → bits 1 and 3 → 0b1010 +/// assert_eq!(words[0], 0b1010); +/// ``` +#[inline] +pub fn ne_u32_to_mask(values: &[u32], needle: u32, out_words: &mut [u64]) { + eq_u32_to_mask(values, needle, out_words); + for w in out_words.iter_mut() { + *w = !*w; + } + clear_mask_tail(out_words, values.len()); +} + +/// `dst = !src` over `n_rows` elements — the tail-aware complement. Bits at +/// or past `n_rows` are written `0`, so a conforming input yields a +/// conforming output (plain `!` on the words would set every tail bit). +/// +/// # Panics +/// +/// Panics unless `src.len() == dst.len() >= n_rows.div_ceil(64)`. +/// +/// # Examples +/// +/// ``` +/// use ndarray::simd::mask_not; +/// +/// let src = [0b0011u64]; +/// let mut dst = [0u64; 1]; +/// mask_not(&src, 4, &mut dst); +/// // complement of the low 4 bits of 0b0011, tail past row 4 stays zero +/// assert_eq!(dst[0], 0b1100); +/// ``` +#[inline] +pub fn mask_not(src: &[u64], n_rows: usize, dst: &mut [u64]) { + assert_eq!(src.len(), dst.len(), "mask_not: src/dst length mismatch"); + assert!( + dst.len() >= mask_words_for(n_rows), + "mask_not: dst.len()={} < required {}", + dst.len(), + mask_words_for(n_rows) + ); + for (d, &s) in dst.iter_mut().zip(src.iter()) { + *d = !s; + } + clear_mask_tail(dst, n_rows); +} + +/// `dst = !dst` over `n_rows` elements, in place. Same tail contract as +/// [`mask_not`]. +/// +/// # Panics +/// +/// Panics if `dst.len() < n_rows.div_ceil(64)`. +/// +/// # Examples +/// +/// ``` +/// use ndarray::simd::mask_not_assign; +/// +/// let mut dst = [0b0011u64]; +/// mask_not_assign(&mut dst, 4); +/// // complement of the low 4 bits of 0b0011, tail past row 4 stays zero +/// assert_eq!(dst[0], 0b1100); +/// ``` +#[inline] +pub fn mask_not_assign(dst: &mut [u64], n_rows: usize) { + assert!( + dst.len() >= mask_words_for(n_rows), + "mask_not_assign: dst.len()={} < required {}", + dst.len(), + mask_words_for(n_rows) + ); + for d in dst.iter_mut() { + *d = !*d; + } + clear_mask_tail(dst, n_rows); +} + +/// `dst = a ^ b`, elementwise over `u64` mask words — symmetric difference. +/// XOR preserves the trailing-zero guarantee iff both inputs conform +/// (`0 ^ 0 = 0`). Its own primitive, with its own realization on every +/// backend (`U64x8: BitXor`), never spelled as a three-input table. +/// +/// # Panics +/// +/// Panics unless `a.len() == b.len() == dst.len()`. +/// +/// # Examples +/// +/// ``` +/// use ndarray::simd::mask_xor; +/// +/// let a = [0b0110u64]; +/// let b = [0b0011u64]; +/// let mut dst = [0u64; 1]; +/// mask_xor(&a, &b, &mut dst); +/// // bits set in exactly one of a, b: bit 1 (both) cancels, 0 and 2 survive +/// assert_eq!(dst[0], 0b0101); +/// ``` +#[inline] +pub fn mask_xor(a: &[u64], b: &[u64], dst: &mut [u64]) { + assert_eq!(a.len(), b.len(), "mask_xor: a/b length mismatch"); + assert_eq!(a.len(), dst.len(), "mask_xor: a/dst length mismatch"); + const L: usize = crate::simd::U64x8::LANES; + let (ca, ta) = a.as_chunks::(); + let (cb, tb) = b.as_chunks::(); + let (cd, td) = dst.as_chunks_mut::(); + for ((x, y), d) in ca.iter().zip(cb).zip(cd.iter_mut()) { + let va = crate::simd::U64x8::from_array(*x); + let vb = crate::simd::U64x8::from_array(*y); + *d = (va ^ vb).to_array(); + } + if !ta.is_empty() { + let va = crate::simd::U64x8::from_array(pad_tail(ta)); + let vb = crate::simd::U64x8::from_array(pad_tail(tb)); + td.copy_from_slice(&(va ^ vb).to_array()[..td.len()]); + } +} + +/// `dst ^= src`, in place. Same tail contract as [`mask_xor`]. +/// +/// # Panics +/// +/// Panics if `dst.len() != src.len()`. +/// +/// # Examples +/// +/// ``` +/// use ndarray::simd::mask_xor_assign; +/// +/// let mut dst = [0b0110u64]; +/// let src = [0b0011u64]; +/// mask_xor_assign(&mut dst, &src); +/// // bit 1 (set in both) cancels, bits 0 and 2 survive +/// assert_eq!(dst[0], 0b0101); +/// ``` +#[inline] +pub fn mask_xor_assign(dst: &mut [u64], src: &[u64]) { + assert_eq!(dst.len(), src.len(), "mask_xor_assign: length mismatch"); + const L: usize = crate::simd::U64x8::LANES; + let (cd, td) = dst.as_chunks_mut::(); + let (cs, ts) = src.as_chunks::(); + for (d, s) in cd.iter_mut().zip(cs) { + let vd = crate::simd::U64x8::from_array(*d); + let vs = crate::simd::U64x8::from_array(*s); + *d = (vd ^ vs).to_array(); + } + if !td.is_empty() { + let vd = crate::simd::U64x8::from_array(pad_tail(td)); + let vs = crate::simd::U64x8::from_array(pad_tail(ts)); + td.copy_from_slice(&(vd ^ vs).to_array()[..td.len()]); + } +} + +/// `true` iff any bit is set. Word-OR reduction; an empty slice is `false`. +/// This is the survivor test a fused plan uses to stop early, and the +/// `EXISTS` terminal. +/// +/// Reads EVERY word, surplus words included, and takes no `n_rows` — it +/// relies on the normative contract that every writer leaves bits at or past +/// the element count zero. A destination that was written by something +/// outside this module with a dirty tail will read as "some row set". The +/// pair [`mask_all`] takes `n_rows` because a full-population test must know +/// where the population ends; a non-empty test does not. +/// +/// # Examples +/// +/// ``` +/// use ndarray::simd::mask_any; +/// +/// // all-zero word: nothing set +/// assert!(!mask_any(&[0u64])); +/// // bit 2 set (0b0100): at least one row selected +/// assert!(mask_any(&[0b0100u64])); +/// ``` +#[inline] +pub fn mask_any(words: &[u64]) -> bool { + let mut acc = 0u64; + for &w in words { + acc |= w; + } + acc != 0 +} + +/// `true` iff every one of the first `n_rows` bits is set (a conforming mask +/// whose population is the whole universe). `n_rows == 0` is vacuously +/// `true`. Surplus words past the live range are ignored. +/// +/// # Panics +/// +/// Panics if `words.len() < n_rows.div_ceil(64)`. +/// +/// # Examples +/// +/// ``` +/// use ndarray::simd::mask_all; +/// +/// // bits 0, 1, 2 set (0b0111): rows 0..3 are all selected, row 3 is not +/// let words = [0b0111u64]; +/// assert!(mask_all(&words, 3)); +/// assert!(!mask_all(&words, 4)); +/// ``` +#[inline] +pub fn mask_all(words: &[u64], n_rows: usize) -> bool { + let full = n_rows / 64; + assert!( + words.len() >= mask_words_for(n_rows), + "mask_all: words.len()={} < required {}", + words.len(), + mask_words_for(n_rows) + ); + let mut acc = u64::MAX; + for &w in &words[..full] { + acc &= w; + } + if acc != u64::MAX { + return false; + } + let rem = n_rows % 64; + rem == 0 || (words[full] & ((1u64 << rem) - 1)) == (1u64 << rem) - 1 +} + +/// Packs `((values[i] ^ pattern) & care) == 0` — equality on the bits `care` +/// selects, "don't care" elsewhere — into `out_words`. `care == 0` matches +/// every element; `care == u32::MAX` is exact equality. Full overwrite, +/// trailing bits zero. Lowered as ONE ternlog per 16 lanes +/// ([`crate::simd::ternlog::XOR_AND`] = `(a ^ b) & c`) followed by an +/// `eq_bitmask` against zero. +/// +/// # Panics +/// +/// Panics if `out_words.len() < values.len().div_ceil(64)`. +/// +/// # Examples +/// +/// ``` +/// use ndarray::simd::ternary_match_u32_to_mask; +/// +/// let values = [0b1010u32, 0b1110, 0b0010, 0b1011]; +/// let mut words = [0u64; 1]; +/// // pattern 0b1010 with bit 2 "don't care" (care=0b1011, bit 2 clear): +/// // elements 0 and 1 match on every cared-about bit +/// ternary_match_u32_to_mask(&values, 0b1010, 0b1011, &mut words); +/// assert_eq!(words[0], 0b0011); +/// ``` +#[inline] +pub fn ternary_match_u32_to_mask(values: &[u32], pattern: u32, care: u32, out_words: &mut [u64]) { + let n = values.len(); + let words = mask_words_for(n); + assert!( + out_words.len() >= words, + "ternary_match_u32_to_mask: out_words.len()={} < required {}", + out_words.len(), + words + ); + for w in out_words.iter_mut() { + *w = 0; + } + let p = crate::simd::U32x16::splat(pattern); + let c = crate::simd::U32x16::splat(care); + let zero = crate::simd::U32x16::splat(0); + let (chunks, tail) = values.as_chunks::<16>(); + for (g, chunk) in chunks.iter().enumerate() { + let v = crate::simd::U32x16::from_array(*chunk); + let bits = v + .ternlog::<{ crate::simd::ternlog::XOR_AND }>(p, c) + .eq_bitmask(zero); + out_words[g / 4] |= (bits as u64) << ((g % 4) * 16); + } + if !tail.is_empty() { + let g = chunks.len(); + let bits = crate::simd::U32x16::from_array(pad_tail(tail)) + .ternlog::<{ crate::simd::ternlog::XOR_AND }>(p, c) + .eq_bitmask(zero) + & tail_lane_bits(tail.len()); + out_words[g / 4] |= (bits as u64) << ((g % 4) * 16); + } +} + +/// The 64-bit sibling of [`ternary_match_u32_to_mask`]: packs +/// `((values[i] ^ pattern) & care) == 0`. Full overwrite, trailing bits zero. +/// Lowered as one ternlog per 8 lanes plus a per-lane zero test. +/// +/// # Panics +/// +/// Panics if `out_words.len() < values.len().div_ceil(64)`. +/// +/// # Examples +/// +/// ``` +/// use ndarray::simd::ternary_match_u64_to_mask; +/// +/// let values = [0b1010u64, 0b1110, 0b0010, 0b1011]; +/// let mut words = [0u64; 1]; +/// // same pattern/care as the u32 sibling: elements 0 and 1 match +/// ternary_match_u64_to_mask(&values, 0b1010, 0b1011, &mut words); +/// assert_eq!(words[0], 0b0011); +/// ``` +#[inline] +pub fn ternary_match_u64_to_mask(values: &[u64], pattern: u64, care: u64, out_words: &mut [u64]) { + let n = values.len(); + let words = mask_words_for(n); + assert!( + out_words.len() >= words, + "ternary_match_u64_to_mask: out_words.len()={} < required {}", + out_words.len(), + words + ); + for w in out_words.iter_mut() { + *w = 0; + } + let p = crate::simd::U64x8::splat(pattern); + let c = crate::simd::U64x8::splat(care); + const L: usize = crate::simd::U64x8::LANES; + let (chunks, tail) = values.as_chunks::(); + for (g, chunk) in chunks.iter().enumerate() { + let r = crate::simd::U64x8::from_array(*chunk) + .ternlog::<{ crate::simd::ternlog::XOR_AND }>(p, c) + .to_array(); + let mut bits = 0u64; + for (lane, &x) in r.iter().enumerate() { + bits |= ((x == 0) as u64) << lane; + } + out_words[g / 8] |= bits << ((g % 8) * 8); + } + if !tail.is_empty() { + let g = chunks.len(); + let r = crate::simd::U64x8::from_array(pad_tail(tail)) + .ternlog::<{ crate::simd::ternlog::XOR_AND }>(p, c) + .to_array(); + let mut bits = 0u64; + for (lane, &x) in r.iter().take(tail.len()).enumerate() { + bits |= ((x == 0) as u64) << lane; + } + out_words[g / 8] |= bits << ((g % 8) * 8); + } +} + +/// Care-masked match of a **12-byte little-endian register** found at +/// `first_offset + i * stride_bytes` for `i in 0..count` — the strided AoS +/// form for a V3 facet (`4-byte classid | 12-byte payload`: point +/// `first_offset` at the payload) inside a 16-byte facet or a 512-byte row. +/// Element `i` matches iff every byte `k` satisfies +/// `(reg[k] ^ pattern[k]) & care[k] == 0`. Full overwrite, trailing bits zero. +/// +/// Loads are `from_le_bytes` over byte slices (no alignment requirement). +/// The compare is vectorised as `(lo64, hi32)` — 8 registers per `U64x8` +/// ternlog for the low 8 bytes, 16 per `U32x16` ternlog for the high 4 — the +/// gathers are scalar, exactly as [`eq_u32_strided_to_mask`] documents (at +/// row strides ≥ a cache line the walk is memory-bound and a gather buys +/// nothing). +/// +/// # Panics +/// +/// Panics if `out_words.len() < count.div_ceil(64)`, or if any element's +/// 12 bytes would fall outside `bytes` (checked up front with overflow-safe +/// arithmetic; the loop never reads out of bounds). +/// +/// # Examples +/// +/// ``` +/// use ndarray::simd::ternary_match_strided_to_mask; +/// +/// // Two 16-byte records; only the register's first byte is "cared about". +/// let mut bytes = vec![0u8; 32]; +/// bytes[0] = 0xAA; // record 0's cared byte matches the pattern +/// bytes[16] = 0xBB; // record 1's cared byte does not +/// let mut pattern = [0u8; 12]; +/// pattern[0] = 0xAA; +/// let mut care = [0u8; 12]; +/// care[0] = 0xFF; // bytes 1..12 are don't-care +/// let mut words = [0u64; 1]; +/// ternary_match_strided_to_mask(&bytes, 0, 16, 2, &pattern, &care, &mut words); +/// assert_eq!(words[0], 0b01); // only record 0 matches +/// ``` +#[inline] +pub fn ternary_match_strided_to_mask( + bytes: &[u8], first_offset: usize, stride_bytes: usize, count: usize, pattern: &[u8; 12], care: &[u8; 12], + out_words: &mut [u64], +) { + let words = mask_words_for(count); + assert!( + out_words.len() >= words, + "ternary_match_strided_to_mask: out_words.len()={} < required {}", + out_words.len(), + words + ); + if count > 0 { + let last_end = (count - 1) + .checked_mul(stride_bytes) + .and_then(|x| x.checked_add(first_offset)) + .and_then(|x| x.checked_add(12)) + .expect("ternary_match_strided_to_mask: offset arithmetic overflow"); + assert!( + last_end <= bytes.len(), + "ternary_match_strided_to_mask: last element ends at {last_end} > bytes.len() {}", + bytes.len() + ); + } + for w in out_words.iter_mut() { + *w = 0; + } + let plo = u64::from_le_bytes(pattern[0..8].try_into().expect("8 bytes")); + let clo = u64::from_le_bytes(care[0..8].try_into().expect("8 bytes")); + let phi = u32::from_le_bytes(pattern[8..12].try_into().expect("4 bytes")); + let chi = u32::from_le_bytes(care[8..12].try_into().expect("4 bytes")); + let vplo = crate::simd::U64x8::splat(plo); + let vclo = crate::simd::U64x8::splat(clo); + let vphi = crate::simd::U32x16::splat(phi); + let vchi = crate::simd::U32x16::splat(chi); + let zero32 = crate::simd::U32x16::splat(0); + let groups = count / 16; + let mut lo = [0u64; 16]; + let mut hi = [0u32; 16]; + for g in 0..groups { + for k in 0..16 { + let o = first_offset + (g * 16 + k) * stride_bytes; + lo[k] = u64::from_le_bytes(bytes[o..o + 8].try_into().expect("8 bytes")); + hi[k] = u32::from_le_bytes(bytes[o + 8..o + 12].try_into().expect("4 bytes")); + } + let hi_bits = crate::simd::U32x16::from_array(hi) + .ternlog::<{ crate::simd::ternlog::XOR_AND }>(vphi, vchi) + .eq_bitmask(zero32); + let mut lo_bits = 0u16; + for (half, arr) in lo.as_chunks::<8>().0.iter().enumerate() { + let r = crate::simd::U64x8::from_array(*arr) + .ternlog::<{ crate::simd::ternlog::XOR_AND }>(vplo, vclo) + .to_array(); + for (lane, &x) in r.iter().enumerate() { + lo_bits |= ((x == 0) as u16) << (half * 8 + lane); + } + } + let bits = hi_bits & lo_bits; + out_words[g / 4] |= (bits as u64) << ((g % 4) * 16); + } + for i in (groups * 16)..count { + let o = first_offset + i * stride_bytes; + let l = u64::from_le_bytes(bytes[o..o + 8].try_into().expect("8 bytes")); + let h = u32::from_le_bytes(bytes[o + 8..o + 12].try_into().expect("4 bytes")); + if (l ^ plo) & clo == 0 && (h ^ phi) & chi == 0 { + out_words[i / 64] |= 1u64 << (i % 64); + } + } +} + +/// Minimum of `values[i]` over set mask bits, `None` when no bit is set. +/// Same bit order and "bits at or past `values.len()` are ignored" contract +/// as [`masked_sum_i32`]; cost proportional to the popcount. +/// +/// # Panics +/// +/// Panics if `mask_words.len() < values.len().div_ceil(64)`. +/// +/// # Examples +/// +/// ``` +/// use ndarray::simd::masked_min_i32; +/// +/// let values = [10i32, -5, 30, 2]; +/// // bits 1 and 3 select -5 and 2; the minimum of the two is -5 +/// assert_eq!(masked_min_i32(&values, &[0b1010]), Some(-5)); +/// ``` +#[inline] +pub fn masked_min_i32(values: &[i32], mask_words: &[u64]) -> Option { + masked_fold_i32(values, mask_words, i32::min) +} + +/// Maximum of `values[i]` over set mask bits, `None` when no bit is set. +/// Contract as [`masked_min_i32`]. +/// +/// # Panics +/// +/// Panics if `mask_words.len() < values.len().div_ceil(64)`. +/// +/// # Examples +/// +/// ``` +/// use ndarray::simd::masked_max_i32; +/// +/// let values = [10i32, -5, 30, 2]; +/// // bits 1 and 3 select -5 and 2; the maximum of the two is 2 +/// assert_eq!(masked_max_i32(&values, &[0b1010]), Some(2)); +/// ``` +#[inline] +pub fn masked_max_i32(values: &[i32], mask_words: &[u64]) -> Option { + masked_fold_i32(values, mask_words, i32::max) +} + +#[inline(always)] +fn masked_fold_i32(values: &[i32], mask_words: &[u64], f: impl Fn(i32, i32) -> i32) -> Option { + let n = values.len(); + let words = mask_words_for(n); + assert!( + mask_words.len() >= words, + "masked_fold_i32: mask_words.len()={} < required {}", + mask_words.len(), + words + ); + let mut acc: Option = None; + for (w, &word) in mask_words.iter().take(words).enumerate() { + let base = w * 64; + let mut bits = word; + let valid = n - base; + if valid < 64 { + bits &= (1u64 << valid) - 1; + } + while bits != 0 { + let i = base + bits.trailing_zeros() as usize; + bits &= bits - 1; + acc = Some(match acc { + None => values[i], + Some(a) => f(a, values[i]), + }); + } + } + acc +} + +/// `dst[i] = if mask bit i { a[i] } else { b[i] }` — the conditional-select +/// (`CASE WHEN`) over a row mask, with no compaction. Bits at or past +/// `a.len()` are ignored. Plain index loop, deliberately: a bit-per-element +/// select over `i32` has no lane wrapper in the mask vocabulary yet, and a +/// scalar loop is the honest shape until one is measured to be needed +/// (no assembly inspection backs a claim about what LLVM emits here). +/// +/// # Panics +/// +/// Panics unless `a.len() == b.len() == dst.len()` and +/// `mask_words.len() >= a.len().div_ceil(64)`. +/// +/// # Examples +/// +/// ``` +/// use ndarray::simd::blend_i32; +/// +/// let a = [1i32, 2, 3, 4]; +/// let b = [10i32, 20, 30, 40]; +/// let mut dst = [0i32; 4]; +/// // bits 0 and 2 (0b0101) pick from `a`; bits 1 and 3 pick from `b` +/// blend_i32(&[0b0101], &a, &b, &mut dst); +/// assert_eq!(dst, [1, 20, 3, 40]); +/// ``` +#[inline] +pub fn blend_i32(mask_words: &[u64], a: &[i32], b: &[i32], dst: &mut [i32]) { + assert_eq!(a.len(), b.len(), "blend_i32: a/b length mismatch"); + assert_eq!(a.len(), dst.len(), "blend_i32: a/dst length mismatch"); + let n = a.len(); + assert!( + mask_words.len() >= mask_words_for(n), + "blend_i32: mask_words.len()={} < required {}", + mask_words.len(), + mask_words_for(n) + ); + for i in 0..n { + let bit = (mask_words[i / 64] >> (i % 64)) & 1; + dst[i] = if bit == 1 { a[i] } else { b[i] }; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // ── Packed-bitmask predicates + mask algebra ──────────────────────────── + // + // Every test compares the shipped path against an INDEPENDENT scalar + // reference written inline here (never against the implementation's own + // scalar tail, which would be tautological), over a fixed-seed corpus plus + // the explicit edge cases: empty, 1, 63, 64, 65, non-multiples of 64, + // all-match, no-match, `u32::MAX` needle, `i32::MIN`/`i32::MAX` thresholds, + // and negative values. Bit order and the trailing-zero guarantee are + // asserted literally, against hand-computed `u64` words. + // + // Dispatch is compile-time, so one build exercises one backend; the + // scalar references below are what makes "all backends agree" checkable by + // re-running under `-Ctarget-cpu=x86-64-v3` (AVX2 arm) and + // `-Ctarget-cpu=x86-64-v4` (AVX-512 arm). + + /// Deterministic fixed-seed PRNG (SplitMix64) — no dev-dependency needed + /// and the corpus is byte-identical on every run and every backend. + fn splitmix64(state: &mut u64) -> u64 { + *state = state.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = *state; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } + + /// Independent reference: bit `i % 64` of word `i / 64` set where the + /// predicate holds, everything else zero. + fn ref_pack(values: &[T], n_words: usize, pred: impl Fn(T) -> bool) -> Vec { + let mut words = vec![0u64; n_words]; + for (i, &v) in values.iter().enumerate() { + if pred(v) { + words[i / 64] |= 1u64 << (i % 64); + } + } + words + } + + /// Lengths that straddle every boundary that matters: word edges (63/64/65), + /// the 16-lane group edge (15/16/17), and non-multiples of both. + const MASK_LENS: &[usize] = &[0, 1, 2, 15, 16, 17, 31, 32, 33, 47, 63, 64, 65, 100, 127, 128, 129, 200, 255, 256]; + + #[test] + fn eq_u32_to_mask_matches_scalar_reference() { + for &len in MASK_LENS { + let mut seed = 0xA5A5_1234_DEAD_BEEF; + let values: Vec = (0..len) + .map(|_| (splitmix64(&mut seed) % 7) as u32) + .collect(); + + for needle in [0u32, 1, 3, 6, 42, u32::MAX] { + let n_words = len.div_ceil(64); + let expected = ref_pack(&values, n_words, |v| v == needle); + + let mut got = vec![0u64; n_words]; + eq_u32_to_mask(&values, needle, &mut got); + assert_eq!(got, expected, "eq_u32_to_mask len={len} needle={needle}"); + } + } + } + + #[test] + fn eq_u32_to_mask_all_match_and_no_match() { + for &len in MASK_LENS { + let n_words = len.div_ceil(64); + + // All-match: every in-range bit set, every out-of-range bit clear. + let all = vec![9u32; len]; + let mut got = vec![0u64; n_words]; + eq_u32_to_mask(&all, 9, &mut got); + assert_eq!(got, ref_pack(&all, n_words, |v| v == 9), "all-match len={len}"); + // Independent cross-check on the count, so a wrong-but-consistent + // reference cannot hide: exactly `len` bits, no more. + let popcnt: u32 = got.iter().map(|w| w.count_ones()).sum(); + assert_eq!(popcnt as usize, len, "all-match popcount len={len}"); + + // No-match: strictly zero everywhere. + let mut got = vec![u64::MAX; n_words]; // pre-dirtied — must be overwritten + eq_u32_to_mask(&all, 10, &mut got); + assert!(got.iter().all(|&w| w == 0), "no-match must be all zeros, len={len}"); + } + } + + #[test] + fn eq_u32_to_mask_u32_max_needle_and_values() { + // u32::MAX is both a legal needle and a legal value; neither is special. + let values = [u32::MAX, 0, u32::MAX, 1, u32::MAX - 1]; + let mut got = [0u64; 1]; + eq_u32_to_mask(&values, u32::MAX, &mut got); + assert_eq!(got[0], 0b00101, "u32::MAX needle → bits 0 and 2"); + + eq_u32_to_mask(&values, u32::MAX - 1, &mut got); + assert_eq!(got[0], 0b10000, "u32::MAX-1 needle → bit 4 only"); + } + + /// The strided primitive against an independent reference, over an + /// AoS-facet buffer shape (u32 field at `first_offset` inside a + /// `stride_bytes`-wide row). Strides cover the contiguous case (4), a + /// facet within a 16-byte record, and a 512-byte row. + #[test] + fn eq_u32_strided_to_mask_matches_scalar_reference() { + for &count in MASK_LENS { + for &(first_offset, stride) in &[(0usize, 4usize), (4, 16), (16, 512), (0, 0)] { + let byte_len = if count == 0 { + 0 + } else { + first_offset + (count - 1) * stride + 4 + }; + let mut seed = 0x0F0F_CAFE_F00D_1234 ^ (stride as u64); + let mut bytes = vec![0u8; byte_len]; + // Fill every element position with a small-cardinality value so + // needles genuinely hit and miss. stride==0 has ONE position. + let positions = if stride == 0 { count.min(1) } else { count }; + let mut planted = Vec::with_capacity(positions); + for i in 0..positions { + let v = (splitmix64(&mut seed) % 5) as u32; + let off = first_offset + i * stride; + bytes[off..off + 4].copy_from_slice(&v.to_le_bytes()); + planted.push(v); + } + for needle in [0u32, 1, 4, 42] { + let n_words = count.div_ceil(64); + // Independent reference: read back the SAME strided walk + // scalar-only (stride 0 rereads element 0 `count` times). + let logical: Vec = (0..count) + .map(|i| { + if stride == 0 { + planted.first().copied().unwrap_or(0) + } else { + planted[i] + } + }) + .collect(); + let expected = ref_pack(&logical, n_words, |v| v == needle); + + let mut got = vec![u64::MAX; n_words]; // pre-dirtied + eq_u32_strided_to_mask(&bytes, first_offset, stride, count, needle, &mut got); + assert_eq!( + got, expected, + "strided eq count={count} off={first_offset} stride={stride} needle={needle}" + ); + } + } + } + } + + /// Parity with the contiguous primitive: stride 4 over the same values + /// must produce bit-identical masks — two independent implementations of + /// one specification. + #[test] + fn eq_u32_strided_stride4_matches_contiguous_primitive() { + for &count in MASK_LENS { + let mut seed = 0xBEE5_0000_0000_0001; + let values: Vec = (0..count) + .map(|_| (splitmix64(&mut seed) % 9) as u32) + .collect(); + let bytes: Vec = values.iter().flat_map(|v| v.to_le_bytes()).collect(); + let n_words = count.div_ceil(64); + let mut a = vec![0u64; n_words]; + let mut b = vec![0u64; n_words]; + for needle in [0u32, 3, 8, u32::MAX] { + eq_u32_to_mask(&values, needle, &mut a); + eq_u32_strided_to_mask(&bytes, 0, 4, count, needle, &mut b); + assert_eq!(a, b, "contiguous vs strided count={count} needle={needle}"); + } + } + } + + #[test] + #[should_panic(expected = "out of bounds")] + fn eq_u32_strided_rejects_a_last_element_past_the_buffer() { + // 3 elements at stride 16 need bytes 32..36; a 35-byte buffer is short. + let bytes = vec![0u8; 35]; + let mut words = [0u64; 1]; + eq_u32_strided_to_mask(&bytes, 0, 16, 3, 7, &mut words); + } + + #[test] + #[should_panic(expected = "offset arithmetic overflow")] + fn eq_u32_strided_rejects_overflowing_offset_arithmetic() { + let bytes = vec![0u8; 64]; + let mut words = [0u64; 1]; + // (count-1) * stride overflows usize — must panic, not wrap into a + // bogus in-bounds read. + eq_u32_strided_to_mask(&bytes, 0, usize::MAX, 3, 7, &mut words); + } + + #[test] + fn eq_u32_strided_empty_count_writes_only_zeros() { + let bytes: Vec = Vec::new(); + let mut words = [u64::MAX; 2]; + eq_u32_strided_to_mask(&bytes, 0, 512, 0, 7, &mut words); + assert_eq!(words, [0, 0], "count=0 must still overwrite the destination"); + } + + #[test] + fn gt_i32_to_mask_matches_scalar_reference() { + for &len in MASK_LENS { + let mut seed = 0x1357_9BDF_0246_8ACE; + // Full signed spread including both extremes, seeded deterministically. + let values: Vec = (0..len) + .map(|i| match i % 11 { + 0 => i32::MIN, + 1 => i32::MAX, + 2 => 0, + 3 => -1, + 4 => 1, + _ => splitmix64(&mut seed) as i32, + }) + .collect(); + + for threshold in [i32::MIN, i32::MIN + 1, -1000, -1, 0, 1, 1000, i32::MAX - 1, i32::MAX] { + let n_words = len.div_ceil(64); + let expected = ref_pack(&values, n_words, |v| v > threshold); + + let mut got = vec![0u64; n_words]; + gt_i32_to_mask(&values, threshold, &mut got); + assert_eq!(got, expected, "gt_i32_to_mask len={len} threshold={threshold}"); + } + } + } + + #[test] + fn gt_i32_to_mask_signed_not_bitwise() { + // The trap: -1 as a bit pattern (0xFFFF_FFFF) is greater than 0 + // unsigned, but -1 > 0 is false. A backend that packed an unsigned + // compare would set bit 1 here. + let values = [5i32, -1, 0, -2_000_000_000, 2_000_000_000]; + let mut got = [0u64; 1]; + gt_i32_to_mask(&values, 0, &mut got); + assert_eq!(got[0], 0b10001, "only +5 and +2e9 exceed 0"); + } + + #[test] + fn gt_i32_to_mask_threshold_extremes() { + let values = [i32::MIN, i32::MIN + 1, 0, i32::MAX - 1, i32::MAX]; + let mut got = [0u64; 1]; + + // i32::MIN threshold: everything strictly greater — all but lane 0. + gt_i32_to_mask(&values, i32::MIN, &mut got); + assert_eq!(got[0], 0b11110, "i32::MIN threshold excludes only i32::MIN itself"); + + // i32::MAX threshold: nothing exceeds it, and `>` is strict so the + // i32::MAX lane itself is clear too. + got[0] = u64::MAX; + gt_i32_to_mask(&values, i32::MAX, &mut got); + assert_eq!(got[0], 0, "nothing exceeds i32::MAX"); + + // i32::MAX - 1 threshold: only i32::MAX. + gt_i32_to_mask(&values, i32::MAX - 1, &mut got); + assert_eq!(got[0], 0b10000, "only i32::MAX exceeds i32::MAX-1"); + } + + /// The real correctness trap: bits past `values.len()` in the last word. + /// A stale high bit would silently inflate every downstream popcount. + #[test] + fn trailing_bits_beyond_len_are_zero() { + for &len in &[1usize, 15, 16, 17, 33, 63, 65, 100, 127, 129, 200] { + let n_words = len.div_ceil(64); + let used = len % 64; // 0 ⇒ the final word is entirely in range + + // Every element matches, so ONLY the out-of-range bits can be zero. + let u = vec![1u32; len]; + let mut got = vec![u64::MAX; n_words + 2]; // pre-dirtied, plus surplus words + eq_u32_to_mask(&u, 1, &mut got); + if used != 0 { + let expected_last = (1u64 << used) - 1; + assert_eq!(got[n_words - 1], expected_last, "eq trailing bits len={len}"); + } else { + assert_eq!(got[n_words - 1], u64::MAX, "eq full final word len={len}"); + } + assert!(got[n_words..].iter().all(|&w| w == 0), "eq surplus words must be zeroed, len={len}"); + + let i = vec![1i32; len]; + let mut got = vec![u64::MAX; n_words + 2]; + gt_i32_to_mask(&i, 0, &mut got); + if used != 0 { + let expected_last = (1u64 << used) - 1; + assert_eq!(got[n_words - 1], expected_last, "gt trailing bits len={len}"); + } else { + assert_eq!(got[n_words - 1], u64::MAX, "gt full final word len={len}"); + } + assert!(got[n_words..].iter().all(|&w| w == 0), "gt surplus words must be zeroed, len={len}"); + } + } + + #[test] + fn empty_input_writes_only_zeros() { + let mut got = [u64::MAX; 3]; + eq_u32_to_mask(&[], 7, &mut got); + assert_eq!(got, [0u64; 3], "empty eq"); + + let mut got = [u64::MAX; 3]; + gt_i32_to_mask(&[], 7, &mut got); + assert_eq!(got, [0u64; 3], "empty gt"); + + // Zero-length destination is legal for a zero-length input. + eq_u32_to_mask(&[], 7, &mut []); + gt_i32_to_mask(&[], 7, &mut []); + + assert_eq!(masked_sum_i32(&[], &[]), 0, "empty masked_sum"); + } + + #[test] + fn single_element_lands_in_bit_zero() { + let mut got = [u64::MAX; 1]; + eq_u32_to_mask(&[7u32], 7, &mut got); + assert_eq!(got[0], 1, "one matching element ⇒ exactly bit 0"); + eq_u32_to_mask(&[8u32], 7, &mut got); + assert_eq!(got[0], 0, "one non-matching element ⇒ no bits"); + } + + /// Bit order asserted against hand-computed literals — the one test that + /// would catch an MSB-first or word-swapped backend, which a + /// reference-vs-implementation comparison alone cannot (both could be + /// wrong the same way if the reference were derived from the code). + #[test] + fn bit_order_is_lsb_first_within_each_word() { + // 130 elements: matches at 0, 1, 63 (word 0 low + high edge), + // 64, 65, 127 (word 1), and 128 (word 2 bit 0). + let matching = [0usize, 1, 63, 64, 65, 127, 128]; + let mut values = vec![0u32; 130]; + for &i in &matching { + values[i] = 1; + } + + let mut got = [0u64; 3]; + eq_u32_to_mask(&values, 1, &mut got); + + assert_eq!(got[0], (1u64 << 0) | (1u64 << 1) | (1u64 << 63), "word 0: elements 0, 1, 63"); + assert_eq!(got[1], (1u64 << 0) | (1u64 << 1) | (1u64 << 63), "word 1: elements 64, 65, 127 → bits 0, 1, 63"); + assert_eq!(got[2], 1u64 << 0, "word 2: element 128 → bit 0, rest zero"); + + // Element 64 is bit 0 of word 1, NOT bit 64-of-something or the high + // bit of word 0 — the word-boundary claim, stated as its own literal. + let mut only_64 = vec![0u32; 130]; + only_64[64] = 1; + let mut got = [0u64; 3]; + eq_u32_to_mask(&only_64, 1, &mut got); + assert_eq!(got, [0u64, 1u64, 0u64], "element 64 ⇒ word 1 bit 0 alone"); + } + + // ── mask algebra ──────────────────────────────────────────────────────── + + #[test] + fn mask_and_or_match_scalar_reference() { + // Lengths straddling the 8-word U64x8 group boundary. + for &len in &[0usize, 1, 2, 7, 8, 9, 15, 16, 17, 31, 63, 64, 100] { + let mut seed = 0xFEED_FACE_CAFE_0001; + let a: Vec = (0..len).map(|_| splitmix64(&mut seed)).collect(); + let b: Vec = (0..len).map(|_| splitmix64(&mut seed)).collect(); + + let ref_and: Vec = a.iter().zip(&b).map(|(x, y)| x & y).collect(); + let ref_or: Vec = a.iter().zip(&b).map(|(x, y)| x | y).collect(); + + let mut dst = vec![0xDEAD_BEEFu64; len]; + mask_and(&a, &b, &mut dst); + assert_eq!(dst, ref_and, "mask_and len={len}"); + + let mut dst = vec![0xDEAD_BEEFu64; len]; + mask_or(&a, &b, &mut dst); + assert_eq!(dst, ref_or, "mask_or len={len}"); + + let mut dst = a.clone(); + mask_and_assign(&mut dst, &b); + assert_eq!(dst, ref_and, "mask_and_assign len={len}"); + + let mut dst = a.clone(); + mask_or_assign(&mut dst, &b); + assert_eq!(dst, ref_or, "mask_or_assign len={len}"); + } + } + + #[test] + fn mask_algebra_identities() { + let a = vec![0x0F0F_0F0F_0F0F_0F0Fu64; 20]; + let zeros = vec![0u64; 20]; + let ones = vec![u64::MAX; 20]; + + let mut dst = vec![1u64; 20]; + mask_and(&a, &ones, &mut dst); + assert_eq!(dst, a, "x & ALL == x"); + + mask_and(&a, &zeros, &mut dst); + assert_eq!(dst, zeros, "x & 0 == 0"); + + mask_or(&a, &zeros, &mut dst); + assert_eq!(dst, a, "x | 0 == x"); + + mask_or(&a, &ones, &mut dst); + assert_eq!(dst, ones, "x | ALL == ALL"); + + // Narrowing: AND is monotone, so the popcount can only shrink. + let mut seed = 0x0BAD_C0DE_0BAD_C0DE; + let b: Vec = (0..20).map(|_| splitmix64(&mut seed)).collect(); + let mut dst = vec![0u64; 20]; + mask_and(&a, &b, &mut dst); + let pc = |w: &[u64]| -> u32 { w.iter().map(|x| x.count_ones()).sum() }; + assert!(pc(&dst) <= pc(&a), "AND narrows"); + assert!(pc(&dst) <= pc(&b), "AND narrows"); + // ...and non-trivially so, or the assertion above is vacuous. + assert!(pc(&dst) < pc(&a), "AND must actually remove bits on this corpus"); + } + + #[test] + #[should_panic(expected = "length mismatch")] + fn mask_and_rejects_length_mismatch() { + let mut dst = [0u64; 4]; + mask_and(&[0u64; 4], &[0u64; 3], &mut dst); + } + + // ── mask_andnot (a & !b) ───────────────────────────────────────────────── + + #[test] + fn mask_andnot_matches_scalar_reference() { + // Same length set as `mask_and_or_match_scalar_reference`, straddling + // the 8-word U64x8 group boundary; len=2 is the `mask_words_for(70)` + // shape (70 rows -> 2 words, a 6-bit tail in the second word). + for &len in &[0usize, 1, 2, 7, 8, 9, 15, 16, 17, 31, 63, 64, 100] { + let mut seed = 0xA11C_E5EE_D000_0001; + let a: Vec = (0..len).map(|_| splitmix64(&mut seed)).collect(); + let b: Vec = (0..len).map(|_| splitmix64(&mut seed)).collect(); + + let ref_andnot: Vec = a.iter().zip(&b).map(|(x, y)| x & !y).collect(); + + let mut dst = vec![0xDEAD_BEEFu64; len]; + mask_andnot(&a, &b, &mut dst); + assert_eq!(dst, ref_andnot, "mask_andnot len={len}"); + + let mut dst = a.clone(); + mask_andnot_assign(&mut dst, &b); + assert_eq!(dst, ref_andnot, "mask_andnot_assign len={len}"); + } + } + + #[test] + fn mask_andnot_algebra_identities() { + let mut seed = 0x1357_9BDF_2468_ACE0; + let a: Vec = (0..20).map(|_| splitmix64(&mut seed)).collect(); + let b: Vec = (0..20).map(|_| splitmix64(&mut seed)).collect(); + + // (a & !b) | (a & b) == a — partitioning a's bits by whether b also + // has them set recovers a exactly. + let mut a_andnot_b = vec![0u64; 20]; + mask_andnot(&a, &b, &mut a_andnot_b); + let mut a_and_b = vec![0u64; 20]; + mask_and(&a, &b, &mut a_and_b); + let mut recombined = vec![0u64; 20]; + mask_or(&a_andnot_b, &a_and_b, &mut recombined); + assert_eq!(recombined, a, "(a & !b) | (a & b) == a"); + + // (a & !b) & b == 0 — the "not b" half can never overlap b. + let mut overlap = vec![0u64; 20]; + mask_and(&a_andnot_b, &b, &mut overlap); + assert_eq!(overlap, vec![0u64; 20], "(a & !b) & b == 0"); + + // ...and non-trivially so: on this corpus a_andnot_b must actually + // differ from a (b removes real bits), or both identities above hold + // vacuously of a no-op. + assert_ne!(a_andnot_b, a, "andnot must actually remove bits on this corpus"); + } + + #[test] + fn mask_andnot_preserves_conforming_tail() { + // 2 words = the `mask_words_for(70)` shape: word 0 fully valid (rows + // 0..63), word 1 valid only in its low 7 bits (rows 64..70); the + // tail is word 1 bits 7..63, which a conforming mask always holds + // zero. + const TAIL_MASK: u64 = !0x7Fu64; // bits 7..63 + + // Arm 1: a conforms (tail zero), b is maximally non-conforming (all + // bits set, including its own tail) — dst must still be zero + // everywhere, tail included, because `a & !b` can never exceed `a`. + let a = [0x1234_5678_9ABC_DEF0u64, 0x0000_0000_0000_005Bu64]; + assert_eq!(a[1] & TAIL_MASK, 0, "fixture precondition: a's tail is zero"); + let b = [u64::MAX; 2]; + let mut dst = [0xDEAD_BEEFu64; 2]; + mask_andnot(&a, &b, &mut dst); + assert_eq!(dst, [0u64, 0u64], "a & !(all-ones) == 0, tail included"); + + // Arm 2: a still conforms; b's body is zero (so it removes nothing + // from a) but b's tail is dirty (all ones) — exactly the shape where + // `!b` flips a normally-zero tail to all ones. dst must equal a + // exactly, and in particular dst's tail must stay zero: a's tail was + // already zero, and `a & !b` can only ever narrow a, never widen it. + let b_dirty_tail = [0u64, TAIL_MASK]; + assert_ne!(b_dirty_tail[1] & TAIL_MASK, 0, "fixture precondition: b's tail is dirty"); + let mut dst = [0xDEAD_BEEFu64; 2]; + mask_andnot(&a, &b_dirty_tail, &mut dst); + assert_eq!(dst, a, "a & !b == a when b's body is 0, even with a dirty b tail"); + assert_eq!(dst[1] & TAIL_MASK, 0, "dst's tail stays zero despite b's dirty tail"); + } + + #[test] + #[should_panic(expected = "length mismatch")] + fn mask_andnot_rejects_length_mismatch() { + let mut dst = [0u64; 4]; + mask_andnot(&[0u64; 4], &[0u64; 3], &mut dst); + } + + #[test] + #[should_panic(expected = "length mismatch")] + fn mask_andnot_assign_rejects_length_mismatch() { + let mut a = [0u64; 4]; + mask_andnot_assign(&mut a, &[0u64; 3]); + } + + // ── mask_ternlog (any 3-input Boolean, one pass) ───────────────────────── + + /// Truth-table reference evaluated one BIT at a time — independent of + /// every backend lane (the scalar tail it once shared with them is gone — + /// tails run through the same packed op as the body). + fn ref_ternlog_bitwise(a: u64, b: u64, c: u64, imm: i32) -> u64 { + let mut r = 0u64; + for bit in 0..64 { + let idx = (((a >> bit) & 1) << 2) | (((b >> bit) & 1) << 1) | ((c >> bit) & 1); + if (imm >> idx) & 1 == 1 { + r |= 1u64 << bit; + } + } + r + } + + /// Exercise one IMM over the family's standard length set, both forms, + /// against the bit-serial reference. + fn check_ternlog_imm() { + for &len in &[0usize, 1, 2, 7, 8, 9, 15, 16, 17, 31, 63, 64, 100] { + let mut seed = 0x7E12_10C0_0000_0001 ^ (IMM as u64); + let a: Vec = (0..len).map(|_| splitmix64(&mut seed)).collect(); + let b: Vec = (0..len).map(|_| splitmix64(&mut seed)).collect(); + let c: Vec = (0..len).map(|_| splitmix64(&mut seed)).collect(); + let expect: Vec = (0..len) + .map(|i| ref_ternlog_bitwise(a[i], b[i], c[i], IMM)) + .collect(); + + let mut dst = vec![0xDEAD_BEEFu64; len]; + mask_ternlog::(&a, &b, &c, &mut dst); + assert_eq!(dst, expect, "mask_ternlog imm={IMM:#04x} len={len}"); + + let mut dst = a.clone(); + mask_ternlog_assign::(&mut dst, &b, &c); + assert_eq!(dst, expect, "mask_ternlog_assign imm={IMM:#04x} len={len}"); + } + } + + #[test] + fn mask_ternlog_matches_bitwise_reference_for_all_256_tables() { + // Const generics need a literal per instantiation; a macro unrolls + // all 256 so no table is left to "obviously the same as the others". + macro_rules! all_imms { + ($($imm:literal),* $(,)?) => { $( check_ternlog_imm::<$imm>(); )* }; + } + all_imms!( + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, 0x10, 0x11, + 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x1F, 0x20, 0x21, 0x22, 0x23, + 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2A, 0x2B, 0x2C, 0x2D, 0x2E, 0x2F, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, + 0x36, 0x37, 0x38, 0x39, 0x3A, 0x3B, 0x3C, 0x3D, 0x3E, 0x3F, 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, + 0x48, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, + 0x5A, 0x5B, 0x5C, 0x5D, 0x5E, 0x5F, 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6A, 0x6B, + 0x6C, 0x6D, 0x6E, 0x6F, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, 0x7B, 0x7C, 0x7D, + 0x7E, 0x7F, 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8A, 0x8B, 0x8C, 0x8D, 0x8E, 0x8F, + 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9A, 0x9B, 0x9C, 0x9D, 0x9E, 0x9F, 0xA0, 0xA1, + 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0xAA, 0xAB, 0xAC, 0xAD, 0xAE, 0xAF, 0xB0, 0xB1, 0xB2, 0xB3, + 0xB4, 0xB5, 0xB6, 0xB7, 0xB8, 0xB9, 0xBA, 0xBB, 0xBC, 0xBD, 0xBE, 0xBF, 0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, + 0xC6, 0xC7, 0xC8, 0xC9, 0xCA, 0xCB, 0xCC, 0xCD, 0xCE, 0xCF, 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, + 0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF, 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 0xE8, 0xE9, + 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0xFA, 0xFB, + 0xFC, 0xFD, 0xFE, 0xFF, + ); + } + + #[test] + fn mask_ternlog_and3_equals_two_and_passes() { + // The consumer motivation: `selected & src & gate` as one AND3 pass + // must be bit-identical to the two-pass `mask_and_assign` spelling. + use crate::simd::ternlog::AND3; + let mut seed = 0xA3D3_0000_0000_0001; + let sel: Vec = (0..20).map(|_| splitmix64(&mut seed)).collect(); + let src: Vec = (0..20).map(|_| splitmix64(&mut seed)).collect(); + let gate: Vec = (0..20).map(|_| splitmix64(&mut seed)).collect(); + + let mut two_pass = sel.clone(); + mask_and_assign(&mut two_pass, &src); + mask_and_assign(&mut two_pass, &gate); + + let mut one_pass = sel.clone(); + mask_ternlog_assign::(&mut one_pass, &src, &gate); + assert_eq!(one_pass, two_pass, "AND3 == and∘and"); + + // Non-vacuous: the narrowing must have removed bits, and both narrower + // operands must have contributed (each alone leaves a different set). + assert_ne!(one_pass, sel, "AND3 must narrow on this corpus"); + let mut src_only = sel.clone(); + mask_and_assign(&mut src_only, &src); + assert_ne!(one_pass, src_only, "gate must contribute, not just src"); + } + + #[test] + fn mask_ternlog_tail_conforms_iff_imm_is_even() { + // 2 words = the `mask_words_for(70)` shape; tail = word 1 bits 7..63. + const TAIL_MASK: u64 = !0x7Fu64; + let a = [0x1234_5678_9ABC_DEF0u64, 0x0000_0000_0000_005Bu64]; + let b = [0x0F0F_0F0F_0F0F_0F0Fu64, 0x0000_0000_0000_0071u64]; + let c = [0xFFFF_0000_FFFF_0000u64, 0x0000_0000_0000_002Eu64]; + for m in [&a, &b, &c] { + assert_eq!(m[1] & TAIL_MASK, 0, "fixture precondition: conforming inputs"); + } + use crate::simd::ternlog::{AND3, MAJ3, OR3, XOR3}; + + // Every named table is even: tail stays zero. + let mut d = [0xDEAD_BEEFu64; 2]; + mask_ternlog::(&a, &b, &c, &mut d); + assert_eq!(d[1] & TAIL_MASK, 0, "AND3 tail"); + mask_ternlog::(&a, &b, &c, &mut d); + assert_eq!(d[1] & TAIL_MASK, 0, "OR3 tail"); + mask_ternlog::(&a, &b, &c, &mut d); + assert_eq!(d[1] & TAIL_MASK, 0, "MAJ3 tail"); + mask_ternlog::(&a, &b, &c, &mut d); + assert_eq!(d[1] & TAIL_MASK, 0, "XOR3 tail"); + + // The can-it-fire half: an ODD table (NOR3 = 0x01, true of all-zero + // inputs) sets every tail bit, so the doc's "iff even" is a real + // boundary and not a restatement of the inputs. + mask_ternlog::<0x01>(&a, &b, &c, &mut d); + assert_eq!(d[1] & TAIL_MASK, TAIL_MASK, "odd IMM fills the tail"); + + // Subset-shaped table against dirty b/c: still a subset of a. + let dirty = [u64::MAX; 2]; + mask_ternlog::(&a, &dirty, &dirty, &mut d); + assert_eq!(d, a, "AND3 against all-ones is a"); + assert_eq!(d[1] & TAIL_MASK, 0, "AND3 tail follows a's tail"); + } + + #[test] + #[should_panic(expected = "length mismatch")] + fn mask_ternlog_rejects_length_mismatch() { + let mut dst = [0u64; 4]; + mask_ternlog::<0x80>(&[0u64; 4], &[0u64; 4], &[0u64; 3], &mut dst); + } + + #[test] + #[should_panic(expected = "length mismatch")] + fn mask_ternlog_assign_rejects_length_mismatch() { + let mut a = [0u64; 4]; + mask_ternlog_assign::<0x80>(&mut a, &[0u64; 3], &[0u64; 4]); + } + + #[test] + #[should_panic(expected = "out_words.len()")] + fn eq_u32_to_mask_rejects_short_destination() { + // 65 elements need 2 words; 1 must be refused, not silently truncated. + let values = vec![0u32; 65]; + let mut got = [0u64; 1]; + eq_u32_to_mask(&values, 0, &mut got); + } + + // ── masked_sum_i32 ────────────────────────────────────────────────────── + + #[test] + fn masked_sum_i32_matches_scalar_reference() { + for &len in MASK_LENS { + let mut seed = 0x2468_ACE0_1357_9BDF; + let values: Vec = (0..len).map(|_| splitmix64(&mut seed) as i32).collect(); + let n_words = len.div_ceil(64); + + for pattern in [0u64, u64::MAX, 0x5555_5555_5555_5555, 0xAAAA_AAAA_AAAA_AAAA, 1] { + let mask = vec![pattern; n_words]; + // Independent reference: widen every selected element to i64. + let expected: i64 = values + .iter() + .enumerate() + .filter(|(i, _)| mask[i / 64] >> (i % 64) & 1 == 1) + .map(|(_, &v)| v as i64) + .sum(); + let got = masked_sum_i32(&values, &mask); + assert_eq!(got, expected, "masked_sum_i32 len={len} pattern={pattern:#x}"); + } + } + } + + #[test] + fn masked_sum_i32_widens_beyond_i32_range() { + // 64 × i32::MAX = 137_438_953_408, which overflows i32 by ~64×. An + // implementation that reduced in i32 (e.g. `I32x16::reduce_sum`) would + // wrap here; the widened contract says it must not. + let values = [i32::MAX; 64]; + let got = masked_sum_i32(&values, &[u64::MAX]); + assert_eq!(got, 64 * i32::MAX as i64); + assert!(got > i32::MAX as i64, "result genuinely exceeds i32 range"); + + // Same on the negative side. + let values = [i32::MIN; 64]; + let got = masked_sum_i32(&values, &[u64::MAX]); + assert_eq!(got, 64 * i32::MIN as i64); + assert!(got < i32::MIN as i64); + } + + #[test] + fn masked_sum_i32_ignores_bits_past_len() { + // 3 elements, an all-ones mask word: bits 3..63 must be ignored, not + // used to index past the slice (which would panic) or counted. + let values = [10i32, 20, 30]; + assert_eq!(masked_sum_i32(&values, &[u64::MAX]), 60); + + // Same across a word boundary: 65 elements, both words all-ones. + let values: Vec = (0..65).collect(); + let expected: i64 = (0..65i64).sum(); + assert_eq!(masked_sum_i32(&values, &[u64::MAX; 2]), expected); + } + + #[test] + fn masked_sum_i32_empty_mask_is_zero() { + let values: Vec = (1..=100).collect(); + assert_eq!(masked_sum_i32(&values, &[0u64; 2]), 0, "no bits set ⇒ 0"); + } + + /// End-to-end composition: the shape the ABI's fused plan runs — two + /// predicates ANDed, then counted and summed. Ties the seven primitives + /// plus `popcount_batch_u64` together on one corpus. + #[test] + fn predicates_compose_into_count_and_sum() { + const N: usize = 1000; + let classes: Vec = (0..N).map(|i| (i % 4) as u32).collect(); + let values: Vec = (0..N).map(|i| i as i32 - 500).collect(); + let n_words = N.div_ceil(64); + + let mut m_class = vec![0u64; n_words]; + eq_u32_to_mask(&classes, 2, &mut m_class); + let mut m_value = vec![0u64; n_words]; + gt_i32_to_mask(&values, 0, &mut m_value); + + let mut acc = vec![u64::MAX; n_words]; + mask_and_assign(&mut acc, &m_class); + mask_and_assign(&mut acc, &m_value); + + // Independent reference over the same predicates. + let want: Vec = (0..N) + .filter(|&i| classes[i] == 2 && values[i] > 0) + .collect(); + let count = crate::bitwise::popcount_batch_u64(&acc); + assert_eq!(count as usize, want.len(), "fused count"); + let sum_ref: i64 = want.iter().map(|&i| values[i] as i64).sum(); + assert_eq!(masked_sum_i32(&values, &acc), sum_ref, "fused sum"); + + // Anti-vacuity: the composition must actually narrow, or this test + // would pass for a no-op AND. `acc` starts as all N rows. + assert!(count > 0, "the fused predicate must select something"); + assert!((count as usize) < N / 4, "the fused predicate must be strictly narrower than either operand"); + } + + // ── masked_strided_group_sum ── + + /// The three groupings of a 12-byte register read the SAME bytes and must + /// give three DIFFERENT answers — otherwise every test below would pass for + /// an implementation that ignored `groups`/`group_bytes`. + #[test] + fn each_grouping_of_the_same_register_reads_it_differently() { + let mut b = vec![0u8; 512]; + for k in 0..12 { + b[4 + k] = (k + 1) as u8; + } + let m = [0b1u64]; + let rails = masked_strided_group_sum(&b, 4, 512, 1, 6, 2, &m).unwrap(); + let trips = masked_strided_group_sum(&b, 4, 512, 1, 4, 3, &m).unwrap(); + let quads = masked_strided_group_sum(&b, 4, 512, 1, 3, 4, &m).unwrap(); + + // Hand-computed from bytes 1..=12, little-endian per group. + assert_eq!(rails, 0x0201 + 0x0403 + 0x0605 + 0x0807 + 0x0A09 + 0x0C0B); + assert_eq!(trips, 0x030201 + 0x060504 + 0x090807 + 0x0C0B0A); + assert_eq!(quads, 0x04030201 + 0x08070605 + 0x0C0B0A09); + assert!(rails != trips && trips != quads && rails != quads); + } + + /// The mask selects records rather than being decoration, and the stride is + /// respected: two records with different content must sum separately and + /// additively. + #[test] + fn the_mask_and_the_stride_both_bind() { + let mut b = vec![0u8; 2 * 64]; + b[0..4].copy_from_slice(&[1, 0, 2, 0]); + b[64..68].copy_from_slice(&[10, 0, 20, 0]); + let f = |m: u64| masked_strided_group_sum(&b, 0, 64, 2, 2, 2, &[m]).unwrap(); + assert_eq!(f(0b00), 0, "an empty mask sums nothing"); + assert_eq!(f(0b01), 3); + assert_eq!(f(0b10), 30); + assert_eq!(f(0b11), 33, "additive over disjoint selections"); + } + + /// A dirty tail bit past `n_records` is ignored rather than read — the + /// buffer here is too short for it, so an unclamped kernel would panic. + #[test] + fn a_dirty_tail_bit_is_ignored() { + let mut b = vec![0u8; 2 * 16]; + b[0..2].copy_from_slice(&[5, 0]); + b[16..18].copy_from_slice(&[7, 0]); + let clean = masked_strided_group_sum(&b, 0, 16, 2, 1, 2, &[0b11]).unwrap(); + let dirty = masked_strided_group_sum(&b, 0, 16, 2, 1, 2, &[0b1111]).unwrap(); + assert_eq!(clean, 12); + assert_eq!(clean, dirty); + } + + /// Overflow is reported, not wrapped. Four max-valued u32 groups per record + /// over many records exceeds `i64::MAX`; the boundary itself is asserted so + /// the claim is checkable rather than narrated. + #[test] + fn overflow_is_reported_rather_than_wrapped() { + let recs = 8usize; + let mut b = vec![0xFFu8; recs * 16]; + let m = [0xFFu64]; + // Small case: comfortably inside i64. + let small = masked_strided_group_sum(&b, 0, 16, recs, 3, 4, &m).unwrap(); + assert_eq!(small, recs as i64 * 3 * 0xFFFF_FFFF); + + // The documented bound, checked: how many max quad records fit? + let per_record = 3i128 * 0xFFFF_FFFFi128; + assert_eq!(i64::MAX as i128 / per_record, 715_827_882); + + // And the range check itself is what decides, not a wrap. + assert!(i64::try_from(i64::MAX as i128 + 1).is_err()); + b.clear(); + } + + #[test] + #[should_panic(expected = "group_bytes")] + fn a_group_wider_than_four_bytes_is_rejected() { + let b = vec![0u8; 64]; + let _ = masked_strided_group_sum(&b, 0, 16, 1, 1, 5, &[0b1]); + } + + #[test] + #[should_panic(expected = "past len")] + fn a_record_reading_past_the_buffer_is_rejected() { + let b = vec![0u8; 8]; + // Record 0's register would read 0..12 out of an 8-byte buffer. + let _ = masked_strided_group_sum(&b, 0, 16, 1, 3, 4, &[0b1]); + } + + // ── 2026-09-13 additions: the closed comparison family, complement/xor/ + // any/all, care-masked register match, masked min/max, blend ── + + /// Deterministic SplitMix64 so every test corpus is reproducible. + fn splitmix(seed: &mut u64) -> u64 { + *seed = seed.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = *seed; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } + + /// Adversarial i32 corpus: boundaries plus randomness, at lengths that + /// straddle the 16-lane group and the 64-bit word. + fn i32_corpus(n: usize, seed: u64) -> Vec { + let mut s = seed; + let edge = [i32::MIN, i32::MIN + 1, -1, 0, 1, 7, i32::MAX - 1, i32::MAX]; + (0..n) + .map(|i| { + if i % 5 == 0 { + edge[(splitmix(&mut s) % 8) as usize] + } else { + splitmix(&mut s) as i32 + } + }) + .collect() + } + + fn scalar_pred_mask(n: usize, pred: impl Fn(usize) -> bool) -> Vec { + let mut m = vec![0u64; n.div_ceil(64)]; + for i in 0..n { + if pred(i) { + m[i / 64] |= 1u64 << (i % 64); + } + } + m + } + + const LENS: [usize; 9] = [0, 1, 15, 16, 17, 63, 64, 65, 1000]; + + #[test] + fn ordered_i32_family_matches_scalar_reference_at_boundaries() { + for &n in &LENS { + let v = i32_corpus(n, 0xC0FFEE); + let mut out = vec![u64::MAX; n.div_ceil(64) + 1]; // dirty, over-long + for &t in &[i32::MIN, i32::MIN + 1, -1, 0, 7, i32::MAX - 1, i32::MAX] { + lt_i32_to_mask(&v, t, &mut out); + assert_eq!(&out[..n.div_ceil(64)], &scalar_pred_mask(n, |i| v[i] < t)[..], "lt n={n} t={t}"); + assert_eq!(out[n.div_ceil(64)], 0, "lt surplus word n={n}"); + ge_i32_to_mask(&v, t, &mut out); + assert_eq!(&out[..n.div_ceil(64)], &scalar_pred_mask(n, |i| v[i] >= t)[..], "ge n={n} t={t}"); + assert_eq!(out[n.div_ceil(64)], 0, "ge surplus word n={n}"); + le_i32_to_mask(&v, t, &mut out); + assert_eq!(&out[..n.div_ceil(64)], &scalar_pred_mask(n, |i| v[i] <= t)[..], "le n={n} t={t}"); + ne_i32_to_mask(&v, t, &mut out); + assert_eq!(&out[..n.div_ceil(64)], &scalar_pred_mask(n, |i| v[i] != t)[..], "ne n={n} t={t}"); + eq_i32_to_mask(&v, t, &mut out); + assert_eq!(&out[..n.div_ceil(64)], &scalar_pred_mask(n, |i| v[i] == t)[..], "eq n={n} t={t}"); + } + } + } + + /// The falsifier for "why not `x > t-1`": at `t == i32::MIN` the shifted + /// form underflows. `lt` must be all-false and `ge` all-true there. + #[test] + fn lt_ge_are_exact_at_i32_min() { + let v = i32_corpus(200, 1); + let mut out = vec![0u64; 4]; + lt_i32_to_mask(&v, i32::MIN, &mut out); + assert!(out.iter().all(|&w| w == 0)); + ge_i32_to_mask(&v, i32::MIN, &mut out); + assert!(mask_all(&out, 200)); + assert_eq!(out[3] >> 8, 0, "tail past 200 must be clear"); + } + + #[test] + fn ne_u32_is_complement_of_eq_with_clean_tail() { + for &n in &LENS { + let v: Vec = (0..n as u32) + .map(|i| if i % 3 == 0 { 7 } else { i }) + .collect(); + let mut e = vec![0u64; n.div_ceil(64)]; + let mut ne = vec![u64::MAX; n.div_ceil(64)]; + eq_u32_to_mask(&v, 7, &mut e); + ne_u32_to_mask(&v, 7, &mut ne); + for w in 0..e.len() { + assert_eq!(e[w] & ne[w], 0, "overlap n={n}"); + } + assert_eq!( + crate::bitwise::popcount_batch_u64(&e) + crate::bitwise::popcount_batch_u64(&ne), + n as u64, + "partition n={n}" + ); + } + } + + #[test] + fn mask_not_clears_the_tail_and_round_trips() { + for &n in &LENS { + let src = scalar_pred_mask(n, |i| i % 3 == 0); + let mut dst = vec![u64::MAX; n.div_ceil(64)]; + mask_not(&src, n, &mut dst); + assert_eq!(dst, scalar_pred_mask(n, |i| i % 3 != 0), "not n={n}"); + mask_not_assign(&mut dst, n); + assert_eq!(dst, src, "double complement n={n}"); + } + // Can-it-fire: a plain `!` would set the tail; the primitive must not. + let src = vec![0u64; 2]; + let mut dst = vec![0u64; 2]; + mask_not(&src, 70, &mut dst); + assert_eq!(dst[0], u64::MAX); + assert_eq!(dst[1], 0b11_1111); + } + + #[test] + fn mask_xor_matches_scalar_and_is_its_own_inverse() { + for &n in &[0usize, 1, 7, 8, 9, 16, 17, 100] { + let mut s = 0xABCDu64; + let a: Vec = (0..n).map(|_| splitmix(&mut s)).collect(); + let b: Vec = (0..n).map(|_| splitmix(&mut s)).collect(); + let mut d = vec![0u64; n]; + mask_xor(&a, &b, &mut d); + let want: Vec = a.iter().zip(&b).map(|(x, y)| x ^ y).collect(); + assert_eq!(d, want, "xor n={n}"); + mask_xor_assign(&mut d, &b); + assert_eq!(d, a, "xor_assign inverse n={n}"); + } + } + + #[test] + fn mask_any_and_all_discriminate() { + assert!(!mask_any(&[])); + assert!(!mask_any(&[0, 0, 0])); + assert!(mask_any(&[0, 0, 1 << 63])); + assert!(mask_all(&[], 0)); + assert!(mask_all(&[u64::MAX, 0b111], 67)); + assert!(!mask_all(&[u64::MAX, 0b011], 67)); + assert!(!mask_all(&[u64::MAX - 1, 0b111], 67)); + assert!(mask_all(&[u64::MAX, u64::MAX], 128)); + assert!(!mask_all(&[u64::MAX, u64::MAX >> 1], 128)); + } + + #[test] + fn ternary_match_u32_u64_match_scalar_and_care_zero_matches_everything() { + for &n in &LENS { + let mut s = 0x5EEDu64; + let v32: Vec = (0..n) + .map(|_| (splitmix(&mut s) as u32) & 0xF0F0_00FF) + .collect(); + let v64: Vec = (0..n) + .map(|_| splitmix(&mut s) & 0xFF00_FF00_0000_FFFF) + .collect(); + let mut out = vec![u64::MAX; n.div_ceil(64) + 1]; + for &(p, c) in + &[(0x1010_0055u32, 0xF0F0_00FFu32), (0, 0), (0x1010_0055, 0xFFFF_FFFF), (0xDEAD_BEEF, 0x0000_00FF)] + { + ternary_match_u32_to_mask(&v32, p, c, &mut out); + assert_eq!( + &out[..n.div_ceil(64)], + &scalar_pred_mask(n, |i| (v32[i] ^ p) & c == 0)[..], + "u32 n={n} p={p:#x} c={c:#x}" + ); + assert_eq!(out[n.div_ceil(64)], 0); + let (p64, c64) = (u64::from(p) << 32 | u64::from(p), u64::from(c) << 16 | u64::from(c)); + ternary_match_u64_to_mask(&v64, p64, c64, &mut out); + assert_eq!( + &out[..n.div_ceil(64)], + &scalar_pred_mask(n, |i| (v64[i] ^ p64) & c64 == 0)[..], + "u64 n={n}" + ); + } + if n > 0 { + ternary_match_u32_to_mask(&v32, 0xFFFF_FFFF, 0, &mut out); + assert!(mask_all(&out, n), "care=0 must match everything n={n}"); + } + } + } + + /// The register-level match over a real facet-shaped buffer: 16-byte + /// facets (classid + 12-byte payload) at stride 16 and at stride 512, with + /// planted hits, a care mask that ignores one byte, and a disable of the + /// care mask that must flip a planted near-miss (the D-MRL-1a falsifier). + #[test] + fn ternary_match_strided_plants_hits_and_care_disable_flips_a_near_miss() { + for &stride in &[16usize, 512] { + for &n in &[1usize, 15, 16, 17, 64, 65, 130] { + let mut bytes = vec![0u8; stride * n + 3]; + let mut s = 0x77u64; + for i in 0..n { + let o = 3 + i * stride + 4; // 3 = deliberate misalignment, +4 = past classid + for k in 0..12 { + bytes[o + k] = splitmix(&mut s) as u8; + } + } + let pattern: [u8; 12] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]; + let care: [u8; 12] = [0xFF; 12]; + let mut care_wild = care; + care_wild[5] = 0; // byte 5 is don't-care + // plant an exact hit at row 0 and a near-miss (byte 5 differs) at the last row + let o0 = 3 + 4; + bytes[o0..o0 + 12].copy_from_slice(&pattern); + let ol = 3 + (n - 1) * stride + 4; + bytes[ol..ol + 12].copy_from_slice(&pattern); + bytes[ol + 5] ^= 0x80; + let mut out = vec![u64::MAX; n.div_ceil(64)]; + ternary_match_strided_to_mask(&bytes, 3 + 4, stride, n, &pattern, &care, &mut out); + let want = scalar_pred_mask(n, |i| { + let o = 3 + i * stride + 4; + (0..12).all(|k| (bytes[o + k] ^ pattern[k]) & care[k] == 0) + }); + assert_eq!(out, want, "exact stride={stride} n={n}"); + if n > 1 { + // At n == 1 the near-miss row IS row 0, so the planted hit + // was deliberately overwritten; only the n > 1 fixtures + // carry both. + assert!(out[0] & 1 == 1, "planted hit at row 0"); + assert_eq!( + (out[(n - 1) / 64] >> ((n - 1) % 64)) & 1, + 0, + "near-miss must NOT match under full care" + ); + } + ternary_match_strided_to_mask(&bytes, 3 + 4, stride, n, &pattern, &care_wild, &mut out); + assert_eq!( + (out[(n - 1) / 64] >> ((n - 1) % 64)) & 1, + 1, + "near-miss MUST match once byte 5 is don't-care" + ); + let want_wild = scalar_pred_mask(n, |i| { + let o = 3 + i * stride + 4; + (0..12).all(|k| (bytes[o + k] ^ pattern[k]) & care_wild[k] == 0) + }); + assert_eq!(out, want_wild, "wild stride={stride} n={n}"); + } + } + } + + #[test] + #[should_panic(expected = "last element ends at")] + fn ternary_match_strided_rejects_a_last_element_past_the_buffer() { + let b = vec![0u8; 16 * 3]; + let mut out = [0u64; 1]; + ternary_match_strided_to_mask(&b, 4, 16, 4, &[0; 12], &[0; 12], &mut out); + } + + #[test] + fn masked_min_max_match_scalar_and_ignore_bits_past_len() { + for &n in &LENS { + let v = i32_corpus(n, 42); + let m = scalar_pred_mask(n, |i| i % 7 < 3); + let mut sel: Vec = (0..n).filter(|&i| i % 7 < 3).map(|i| v[i]).collect(); + sel.sort_unstable(); + assert_eq!(masked_min_i32(&v, &m), sel.first().copied(), "min n={n}"); + assert_eq!(masked_max_i32(&v, &m), sel.last().copied(), "max n={n}"); + assert_eq!(masked_min_i32(&v, &vec![0u64; n.div_ceil(64)]), None); + } + // dirty tail bits past len must be ignored, not read + let v = [5i32, -3, 9]; + assert_eq!(masked_min_i32(&v, &[u64::MAX]), Some(-3)); + assert_eq!(masked_max_i32(&v, &[u64::MAX]), Some(9)); + } + + #[test] + fn blend_i32_selects_by_bit_and_ignores_bits_past_len() { + for &n in &LENS { + let a = i32_corpus(n, 3); + let b = i32_corpus(n, 4); + let m = scalar_pred_mask(n, |i| i % 2 == 0); + let mut d = vec![0i32; n]; + blend_i32(&m, &a, &b, &mut d); + for i in 0..n { + assert_eq!(d[i], if i % 2 == 0 { a[i] } else { b[i] }, "n={n} i={i}"); + } + } + let mut d = [0i32; 3]; + blend_i32(&[u64::MAX], &[1, 2, 3], &[9, 9, 9], &mut d); + assert_eq!(d, [1, 2, 3]); + } + + /// The two new named immediates, pinned against their formulas. + #[test] + fn named_immediates_xor_and_and2_or_match_their_formulas() { + for a in [0u64, 1] { + for b in [0u64, 1] { + for c in [0u64, 1] { + let idx = (a << 2) | (b << 1) | c; + assert_eq!((crate::simd::ternlog::XOR_AND >> idx) & 1, ((a ^ b) & c) as i32, "XOR_AND {a}{b}{c}"); + assert_eq!((crate::simd::ternlog::AND2_OR >> idx) & 1, ((a & b) | c) as i32, "AND2_OR {a}{b}{c}"); + } + } + } + } +} diff --git a/src/simd_neon.rs b/src/simd_neon.rs index 9c95f3e1..43fe225d 100644 --- a/src/simd_neon.rs +++ b/src/simd_neon.rs @@ -478,13 +478,17 @@ pub mod aarch64_simd { use core::fmt; use core::ops::{Add, AddAssign, Div, DivAssign, Mul, MulAssign, Neg, Sub, SubAssign}; - // Integer types come from the scalar fallback in simd.rs — they aren't on - // the perf-critical f32 BLAS-1 / VML path that this module accelerates. - // `U32x16` is the exception: it carries the ARX vocabulary (Add/BitXor/ - // rotate_left) the ChaCha20 lane needs, so it is the native `[U32x4; 4]` - // defined at the top of this file (mirroring `simd_wasm::wasm32_simd`). - pub use super::U32x16; - pub use crate::simd::scalar::{I32x16, U64x8}; + // Three integer types are NATIVE here, defined at the top of this file + // as `[U32x4; 4]` / `[I32x4; 4]` / `[U64x2; 4]` register fan-outs + // (mirroring `simd_wasm::wasm32_simd`): `U32x16` carries the ARX + // vocabulary (Add/BitXor/rotate_left) the ChaCha20 lane needs, and + // `U64x8` / `I32x16` carry the mask family (bulk algebra + ternlog, the + // signed-compare→bitmask family) since the 2026-09-13 five-flavour audit + // — before it, both resolved to the scalar backend on aarch64. The long + // tail of integer lanes (I8x64, U16x32, …) still comes from the scalar + // fallback in simd.rs; none of it is on the f32 BLAS-1 / VML path this + // module accelerates. + pub use super::{I32x16, U32x16, U64x8}; /// 16×f32 backed by 4× NEON `float32x4_t` registers (paired loads). #[derive(Copy, Clone)] @@ -709,7 +713,7 @@ pub mod aarch64_simd { for i in 0..16 { o[i] = a[i] as i32; } - I32x16(o) + I32x16::from_array(o) } } @@ -819,6 +823,27 @@ pub mod aarch64_simd { #[derive(Copy, Clone, Debug)] pub struct F32Mask16(pub u16); impl F32Mask16 { + /// The mask as a packed 16-bit bitmask, LSB-first (bit `i` = lane `i`). + /// The one representation-independent reading of a compare result: every + /// backend stores its mask differently (`__mmask16`, `u16`, + /// `core::simd::Mask`), so callers combine and inspect masks through this + /// rather than the tuple field (the `aabb` broadphase read `.0` directly + /// and did not compile on the portable backend — fixed 2026-09-14). + /// + /// # Examples + /// Bit `i` is lane `i`: with lanes 0 and 15 below the threshold the + /// `simd_lt` mask reads `0b1000_0000_0000_0001`. + /// ```rust,ignore + /// let mut a = [10.0f32; 16]; + /// a[0] = -1.0; + /// a[15] = -1.0; + /// let m = F32x16::from_array(a).simd_lt(F32x16::splat(0.0)); + /// assert_eq!(m.to_bitmask(), 0b1000_0000_0000_0001); + /// ``` + #[inline(always)] + pub fn to_bitmask(self) -> u16 { + self.0 + } #[inline(always)] pub fn select(self, true_val: F32x16, false_val: F32x16) -> F32x16 { let t = true_val.to_array(); @@ -1004,13 +1029,14 @@ pub mod aarch64_simd { for i in 0..8 { o[i] = a[i].to_bits(); } - U64x8(o) + U64x8::from_array(o) } #[inline(always)] pub fn from_bits(bits: U64x8) -> Self { + let b = bits.to_array(); let mut o = [0.0f64; 8]; for i in 0..8 { - o[i] = f64::from_bits(bits.0[i]); + o[i] = f64::from_bits(b[i]); } Self::from_array(o) } @@ -1788,6 +1814,28 @@ impl U32x16 { o } + /// Wrapping horizontal sum (`vaddvq_u32` per quad) — the same method the + /// AVX2 / AVX-512 / scalar `U32x16` carry; it was missing on this backend + /// until the codegen witness (`examples/ternlog_codegen_probe.rs`) failed + /// to compile for aarch64 on 2026-09-14. + /// + /// # Examples + /// ```rust,ignore + /// let v = U32x16::from_array(core::array::from_fn(|i| i as u32)); // 0..16 + /// assert_eq!(v.reduce_sum(), 120); + /// assert_eq!(U32x16::splat(u32::MAX).reduce_sum(), u32::MAX.wrapping_mul(16)); + /// ``` + #[inline(always)] + pub fn reduce_sum(self) -> u32 { + // SAFETY: NEON baseline; register reductions on four owned quads. + let q: [u32; 4] = unsafe { + [vaddvq_u32(self.0[0].0), vaddvq_u32(self.0[1].0), vaddvq_u32(self.0[2].0), vaddvq_u32(self.0[3].0)] + }; + q[0].wrapping_add(q[1]) + .wrapping_add(q[2]) + .wrapping_add(q[3]) + } + /// Lane-wise left-rotate by `n` bits (ARX rotate), fanned over 4 lanes. #[inline(always)] pub fn rotate_left(self, n: u32) -> Self { @@ -1803,8 +1851,8 @@ impl U32x16 { /// /// Bit `i` of the result is set iff `self.lane(i) == other.lane(i)`. Bit /// order is **LSB-first**: lane `0` occupies bit `0`. Same convention as - /// `I32x16::cmpge_zero_mask` / `I32x16::gt_bitmask` (which on aarch64 come - /// from the scalar tier — `I32x16` is re-exported from `simd_scalar`). + /// `I32x16::cmpge_zero_mask` / `I32x16::gt_bitmask` (the native NEON + /// `I32x16` in this file, since the five-flavour audit of #306). /// /// Edge cases: equality is exact bitwise comparison over the full 32-bit /// range, so `u32::MAX` and `0` behave like any other value — no @@ -1881,38 +1929,34 @@ impl U32x16 { #[inline(always)] pub fn ternlog(self, b: Self, c: Self) -> Self { const { assert!(IMM >= 0 && IMM <= 255, "ternlog IMM is an 8-bit truth table") } - let (a, b, c) = (self.to_array(), b.to_array(), c.to_array()); - let mut o = [0u32; 16]; - for i in 0..16 { - let (x, y, z) = (a[i], b[i], c[i]); - let mut r = 0u32; - if IMM & 0x01 != 0 { - r |= !x & !y & !z; - } - if IMM & 0x02 != 0 { - r |= !x & !y & z; - } - if IMM & 0x04 != 0 { - r |= !x & y & !z; - } - if IMM & 0x08 != 0 { - r |= !x & y & z; - } - if IMM & 0x10 != 0 { - r |= x & !y & !z; - } - if IMM & 0x20 != 0 { - r |= x & !y & z; - } - if IMM & 0x40 != 0 { - r |= x & y & !z; - } - if IMM & 0x80 != 0 { - r |= x & y & z; - } - o[i] = r; - } - Self::from_array(o) + // GENERATED lowering (tools/gen_ternlog_bodies.py), per 128-bit quad (NEON). + Self(core::array::from_fn(|p| { + let (x, y, z) = (self.0[p].0, b.0[p].0, c.0[p].0); + let t0: u8 = ((IMM & 1) | ((IMM >> 1) & 2) | ((IMM >> 2) & 4) | ((IMM >> 3) & 8)) as u8; + let t1: u8 = (((IMM >> 1) & 1) | ((IMM >> 2) & 2) | ((IMM >> 3) & 4) | ((IMM >> 4) & 8)) as u8; + // SAFETY: NEON is a baseline feature of every aarch64 target this module compiles + // for; these are pure register operations on values already in `uint32x4_t`. + U32x4(unsafe { + if t0 == t1 { + ternlog_two_input_u32x4(t0, x, y) + } else if t0 == 0 { + vandq_u32(z, ternlog_two_input_u32x4(t1, x, y)) + } else if t1 == 0 { + vbicq_u32(ternlog_two_input_u32x4(t0, x, y), z) + } else if t1 == (t0 ^ 0xF) { + veorq_u32(z, ternlog_two_input_u32x4(t0, x, y)) + } else if t0 == 0xF { + vorrq_u32(vmvnq_u32(z), ternlog_two_input_u32x4(t1, x, y)) + } else if t1 == 0xF { + vorrq_u32(z, ternlog_two_input_u32x4(t0, x, y)) + } else { + vorrq_u32( + vbicq_u32(ternlog_two_input_u32x4(t0, x, y), z), + vandq_u32(ternlog_two_input_u32x4(t1, x, y), z), + ) + } + }) + })) } } @@ -1947,6 +1991,14 @@ impl PartialEq for U32x16 { #[cfg(target_arch = "aarch64")] #[allow(non_camel_case_types)] pub type u32x16 = U32x16; +/// Lowercase alias of the native NEON [`I32x16`] (travels with the type). +#[cfg(target_arch = "aarch64")] +#[allow(non_camel_case_types)] +pub type i32x16 = I32x16; +/// Lowercase alias of the native NEON [`U64x8`] (travels with the type). +#[cfg(target_arch = "aarch64")] +#[allow(non_camel_case_types)] +pub type u64x8 = U64x8; #[cfg(target_arch = "aarch64")] #[derive(Copy, Clone)] @@ -2596,3 +2648,701 @@ mod tests { } } } + +// ═══════════════════════════════════════════════════════════════════════════ +// Native U64x8 / I32x16 — the two lane types the mask family rides +// (2026-09-13, PR #306 five-flavour audit) +// ═══════════════════════════════════════════════════════════════════════════ +// +// `simd_masking_ops` builds every bulk mask op (`mask_and/or/xor/andnot`, +// `mask_ternlog`) on `U64x8` and the whole signed-compare family +// (`gt/lt/ge/le_i32_to_mask`) on `I32x16::gt_bitmask`. Until this section +// both types were re-exported from the SCALAR backend on aarch64, so the mask +// lane ran per-element loops on ARM while only the `U32x16` paths reached +// NEON. Same fan-out shape as `U32x16`: four 128-bit quads, every op applied +// per quad with the NEON intrinsic, ONE narrow `unsafe` per method at the +// intrinsic boundary (the cfg-selected backend file is the capability proof; +// no `#[target_feature]` — operator ruling). Surface = the scalar backend's +// `impl_int_type!` set plus its `U64x8` / `I32x16` extras, signature for +// signature, so nothing that compiled against the scalar re-export changes. + +/// 8×u64 backed by 4× NEON `uint64x2_t` (`[U64x2; 4]`). The packed-mask word +/// lane: `& | ^ !`, `andnot`, `ternlog`, rotates, `popcnt`. +#[cfg(target_arch = "aarch64")] +#[derive(Copy, Clone)] +#[repr(align(64))] +pub struct U64x8(pub [U64x2; 4]); + +#[cfg(target_arch = "aarch64")] +impl Default for U64x8 { + #[inline(always)] + fn default() -> Self { + Self::splat(0) + } +} + +#[cfg(target_arch = "aarch64")] +impl U64x8 { + pub const LANES: usize = 8; + + /// Broadcast `v` to all 8 lanes (`vdupq_n_u64` ×4). + #[inline(always)] + pub fn splat(v: u64) -> Self { + Self([U64x2::splat(v); 4]) + } + + /// All-zero lanes. + #[inline(always)] + pub fn zero() -> Self { + Self::splat(0) + } + + /// Load the first 8 elements of `s` (`vld1q_u64` ×4). Panics if `s.len() < 8`. + #[inline(always)] + pub fn from_slice(s: &[u64]) -> Self { + assert!(s.len() >= 8); + Self([ + U64x2::from_slice(&s[0..2]), + U64x2::from_slice(&s[2..4]), + U64x2::from_slice(&s[4..6]), + U64x2::from_slice(&s[6..8]), + ]) + } + + /// Load from an array (`vld1q_u64` ×4). + #[inline(always)] + pub fn from_array(arr: [u64; 8]) -> Self { + Self::from_slice(&arr) + } + + /// Store to an array (`vst1q_u64` ×4). + #[inline(always)] + pub fn to_array(self) -> [u64; 8] { + let mut arr = [0u64; 8]; + self.copy_to_slice(&mut arr); + arr + } + + /// Store the 8 lanes into the front of `s` (`vst1q_u64` ×4). Panics if `s.len() < 8`. + #[inline(always)] + pub fn copy_to_slice(self, s: &mut [u64]) { + assert!(s.len() >= 8); + self.0[0].copy_to_slice(&mut s[0..2]); + self.0[1].copy_to_slice(&mut s[2..4]); + self.0[2].copy_to_slice(&mut s[4..6]); + self.0[3].copy_to_slice(&mut s[6..8]); + } + + /// Wrapping horizontal sum of all 8 lanes (`vaddvq_u64` per quad). + #[inline(always)] + pub fn reduce_sum(self) -> u64 { + // SAFETY: NEON baseline; register reductions on values already in uint64x2_t. + let q: [u64; 4] = unsafe { + [vaddvq_u64(self.0[0].0), vaddvq_u64(self.0[1].0), vaddvq_u64(self.0[2].0), vaddvq_u64(self.0[3].0)] + }; + q[0].wrapping_add(q[1]) + .wrapping_add(q[2]) + .wrapping_add(q[3]) + } + + /// Lane-wise left-rotate by `n` bits. `n` is taken mod 64. Two `vshlq_u64` + /// (a negative shift count is a right shift on NEON) and one `vorrq_u64` + /// per quad. + #[inline(always)] + pub fn rotate_left(self, n: u32) -> Self { + let n = n % 64; + if n == 0 { + return self; + } + // SAFETY: NEON baseline; pure register ops on uint64x2_t. + Self(core::array::from_fn(|p| unsafe { + let l = vshlq_u64(self.0[p].0, vdupq_n_s64(n as i64)); + let r = vshlq_u64(self.0[p].0, vdupq_n_s64(n as i64 - 64)); + U64x2(vorrq_u64(l, r)) + })) + } + + /// Lane-wise right-rotate by `n` bits — BLAKE2b's direction. + /// `rotr(n) == rotl(64 - n)` exactly. + #[inline(always)] + pub fn rotate_right(self, n: u32) -> Self { + let n = n % 64; + if n == 0 { + return self; + } + self.rotate_left(64 - n) + } + + /// Lane-wise population count: `vcntq_u8` on the bytes, then the + /// `vpaddlq_u8 → vpaddlq_u16 → vpaddlq_u32` widening-add ladder back to + /// one count per u64 lane (0..=64). + #[inline(always)] + pub fn popcnt(self) -> Self { + // SAFETY: NEON baseline; pure register ops. + Self(core::array::from_fn(|p| unsafe { + let bytes = vcntq_u8(vreinterpretq_u8_u64(self.0[p].0)); + U64x2(vpaddlq_u32(vpaddlq_u16(vpaddlq_u8(bytes)))) + })) + } + + /// XOR two vectors lane-wise, popcount each lane, sum all 8 lanes — the + /// Hamming distance of 512 bits. + #[inline(always)] + pub fn xor_popcount(self, other: Self) -> u64 { + (self ^ other).popcnt().reduce_sum() + } + + /// Set difference: `self & !other` (`vbicq_u64` — note the intrinsic's + /// operand order IS `a & !b`, unlike Intel's `andnot`). Same direction as + /// every other backend. + #[inline(always)] + pub fn andnot(self, other: Self) -> Self { + // SAFETY: NEON baseline; pure register ops. + Self(core::array::from_fn(|p| U64x2(unsafe { vbicq_u64(self.0[p].0, other.0[p].0) }))) + } + + /// Any 3-input boolean function of `self`, `b` and `c`, selected by the + /// const truth-table immediate `IMM` — Intel's VPTERNLOG convention + /// (`index = (self << 2) | (b << 1) | c`, result bit = `(IMM >> index) & 1`), + /// matched exactly by every backend. Only `0..=255` is legal (compile-time + /// assert). Named immediates: `crate::simd::ternlog`. The body is + /// generated (`tools/gen_ternlog_bodies.py`): a Shannon-expanded ladder in + /// `vandq/vorrq/veorq/vbicq_u64` per quad. + #[inline(always)] + pub fn ternlog(self, b: Self, c: Self) -> Self { + const { assert!(IMM >= 0 && IMM <= 255, "ternlog IMM is an 8-bit truth table") } + // GENERATED lowering (tools/gen_ternlog_bodies.py), per 128-bit quad (NEON). + Self(core::array::from_fn(|p| { + let (x, y, z) = (self.0[p].0, b.0[p].0, c.0[p].0); + let t0: u8 = ((IMM & 1) | ((IMM >> 1) & 2) | ((IMM >> 2) & 4) | ((IMM >> 3) & 8)) as u8; + let t1: u8 = (((IMM >> 1) & 1) | ((IMM >> 2) & 2) | ((IMM >> 3) & 4) | ((IMM >> 4) & 8)) as u8; + // SAFETY: NEON is a baseline feature of every aarch64 target this module compiles + // for; these are pure register operations on values already in `uint64x2_t`. + U64x2(unsafe { + if t0 == t1 { + ternlog_two_input_u64x2(t0, x, y) + } else if t0 == 0 { + vandq_u64(z, ternlog_two_input_u64x2(t1, x, y)) + } else if t1 == 0 { + vbicq_u64(ternlog_two_input_u64x2(t0, x, y), z) + } else if t1 == (t0 ^ 0xF) { + veorq_u64(z, ternlog_two_input_u64x2(t0, x, y)) + } else if t0 == 0xF { + vorrq_u64(veorq_u64(z, vdupq_n_u64(!0)), ternlog_two_input_u64x2(t1, x, y)) + } else if t1 == 0xF { + vorrq_u64(z, ternlog_two_input_u64x2(t0, x, y)) + } else { + vorrq_u64( + vbicq_u64(ternlog_two_input_u64x2(t0, x, y), z), + vandq_u64(ternlog_two_input_u64x2(t1, x, y), z), + ) + } + }) + })) + } +} + +#[cfg(target_arch = "aarch64")] +impl core::ops::Add for U64x8 { + type Output = Self; + #[inline(always)] + fn add(self, r: Self) -> Self { + Self(core::array::from_fn(|p| self.0[p].add(r.0[p]))) + } +} +#[cfg(target_arch = "aarch64")] +impl core::ops::Sub for U64x8 { + type Output = Self; + #[inline(always)] + fn sub(self, r: Self) -> Self { + Self(core::array::from_fn(|p| self.0[p].sub(r.0[p]))) + } +} +#[cfg(target_arch = "aarch64")] +impl core::ops::AddAssign for U64x8 { + #[inline(always)] + fn add_assign(&mut self, r: Self) { + *self = *self + r; + } +} +#[cfg(target_arch = "aarch64")] +impl core::ops::SubAssign for U64x8 { + #[inline(always)] + fn sub_assign(&mut self, r: Self) { + *self = *self - r; + } +} +#[cfg(target_arch = "aarch64")] +impl core::ops::BitAnd for U64x8 { + type Output = Self; + #[inline(always)] + fn bitand(self, r: Self) -> Self { + // SAFETY: NEON baseline; pure register ops. + Self(core::array::from_fn(|p| U64x2(unsafe { vandq_u64(self.0[p].0, r.0[p].0) }))) + } +} +#[cfg(target_arch = "aarch64")] +impl core::ops::BitOr for U64x8 { + type Output = Self; + #[inline(always)] + fn bitor(self, r: Self) -> Self { + // SAFETY: NEON baseline; pure register ops. + Self(core::array::from_fn(|p| U64x2(unsafe { vorrq_u64(self.0[p].0, r.0[p].0) }))) + } +} +#[cfg(target_arch = "aarch64")] +impl core::ops::BitXor for U64x8 { + type Output = Self; + #[inline(always)] + fn bitxor(self, r: Self) -> Self { + // SAFETY: NEON baseline; pure register ops. + Self(core::array::from_fn(|p| U64x2(unsafe { veorq_u64(self.0[p].0, r.0[p].0) }))) + } +} +#[cfg(target_arch = "aarch64")] +impl core::ops::BitAndAssign for U64x8 { + #[inline(always)] + fn bitand_assign(&mut self, r: Self) { + *self = *self & r; + } +} +#[cfg(target_arch = "aarch64")] +impl core::ops::BitOrAssign for U64x8 { + #[inline(always)] + fn bitor_assign(&mut self, r: Self) { + *self = *self | r; + } +} +#[cfg(target_arch = "aarch64")] +impl core::ops::BitXorAssign for U64x8 { + #[inline(always)] + fn bitxor_assign(&mut self, r: Self) { + *self = *self ^ r; + } +} +#[cfg(target_arch = "aarch64")] +impl core::ops::Not for U64x8 { + type Output = Self; + #[inline(always)] + fn not(self) -> Self { + // SAFETY: NEON baseline; `vmvnq_u32` on the same 128 bits reinterpreted. + Self(core::array::from_fn(|p| { + U64x2(unsafe { vreinterpretq_u64_u32(vmvnq_u32(vreinterpretq_u32_u64(self.0[p].0))) }) + })) + } +} +/// Lane-wise `self << rhs` (per-lane counts; a count of 64 or more yields 0, +/// which is what the shift instruction does — the scalar backend's `<<` +/// panics there in debug builds, so callers already stay inside `0..64`). +#[cfg(target_arch = "aarch64")] +impl core::ops::Shl for U64x8 { + type Output = Self; + #[inline(always)] + fn shl(self, r: Self) -> Self { + // SAFETY: NEON baseline; pure register ops. + Self(core::array::from_fn(|p| U64x2(unsafe { vshlq_u64(self.0[p].0, vreinterpretq_s64_u64(r.0[p].0)) }))) + } +} +/// Lane-wise `self >> rhs` (`vshlq_u64` with negated counts). +#[cfg(target_arch = "aarch64")] +impl core::ops::Shr for U64x8 { + type Output = Self; + #[inline(always)] + fn shr(self, r: Self) -> Self { + // SAFETY: NEON baseline; pure register ops. + Self(core::array::from_fn(|p| { + U64x2(unsafe { vshlq_u64(self.0[p].0, vnegq_s64(vreinterpretq_s64_u64(r.0[p].0))) }) + })) + } +} +#[cfg(target_arch = "aarch64")] +impl PartialEq for U64x8 { + #[inline(always)] + fn eq(&self, other: &Self) -> bool { + // SAFETY: NEON baseline; `vceqq_u64` then an all-lanes check via the u32 min. + (0..4).all(|p| unsafe { vminvq_u32(vreinterpretq_u32_u64(vceqq_u64(self.0[p].0, other.0[p].0))) == u32::MAX }) + } +} +#[cfg(target_arch = "aarch64")] +impl core::fmt::Debug for U64x8 { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "U64x8({:?})", self.to_array()) + } +} + +/// 16×i32 backed by 4× NEON `int32x4_t` (`[I32x4; 4]`). The signed-compare +/// lane: `gt_bitmask` / `cmpge_zero_mask` are `vcgtq_s32` / `vcgezq_s32` with a +/// vector bit-pack (no per-lane loop). +#[cfg(target_arch = "aarch64")] +#[derive(Copy, Clone)] +#[repr(align(64))] +pub struct I32x16(pub [I32x4; 4]); + +#[cfg(target_arch = "aarch64")] +impl Default for I32x16 { + #[inline(always)] + fn default() -> Self { + Self::splat(0) + } +} + +/// Pack a per-lane all-ones/all-zeros `uint32x4_t` compare result into a 4-bit +/// mask, bit `i` = lane `i`: AND with `[1, 2, 4, 8]`, horizontal add. +#[cfg(target_arch = "aarch64")] +#[inline(always)] +fn quad_mask4(cmp: uint32x4_t) -> u16 { + const WEIGHTS: [u32; 4] = [1, 2, 4, 8]; + // SAFETY: NEON baseline. The one memory access is `vld1q_u32` of + // `WEIGHTS`, a `const [u32; 4]` — 16 readable, 4-byte-aligned bytes, which + // is all `vld1q_u32` requires; the rest is register-only. + unsafe { vaddvq_u32(vandq_u32(cmp, vld1q_u32(WEIGHTS.as_ptr()))) as u16 } +} + +#[cfg(target_arch = "aarch64")] +impl I32x16 { + pub const LANES: usize = 16; + + /// Broadcast (`vdupq_n_s32` ×4). + #[inline(always)] + pub fn splat(v: i32) -> Self { + Self([I32x4::splat(v); 4]) + } + + /// All-zero lanes. + #[inline(always)] + pub fn zero() -> Self { + Self::splat(0) + } + + /// Load the first 16 elements of `s` (`vld1q_s32` ×4). Panics if `s.len() < 16`. + #[inline(always)] + pub fn from_slice(s: &[i32]) -> Self { + assert!(s.len() >= 16); + Self([ + I32x4::from_slice(&s[0..4]), + I32x4::from_slice(&s[4..8]), + I32x4::from_slice(&s[8..12]), + I32x4::from_slice(&s[12..16]), + ]) + } + + /// Load from an array (`vld1q_s32` ×4). + #[inline(always)] + pub fn from_array(arr: [i32; 16]) -> Self { + Self::from_slice(&arr) + } + + /// Store to an array (`vst1q_s32` ×4). + #[inline(always)] + pub fn to_array(self) -> [i32; 16] { + let mut arr = [0i32; 16]; + self.copy_to_slice(&mut arr); + arr + } + + /// Store the 16 lanes into the front of `s`. Panics if `s.len() < 16`. + #[inline(always)] + pub fn copy_to_slice(self, s: &mut [i32]) { + assert!(s.len() >= 16); + self.0[0].copy_to_slice(&mut s[0..4]); + self.0[1].copy_to_slice(&mut s[4..8]); + self.0[2].copy_to_slice(&mut s[8..12]); + self.0[3].copy_to_slice(&mut s[12..16]); + } + + /// Wrapping horizontal sum (`vaddvq_s32` per quad). + #[inline(always)] + pub fn reduce_sum(self) -> i32 { + // SAFETY: NEON baseline; register reductions. + let q: [i32; 4] = unsafe { + [vaddvq_s32(self.0[0].0), vaddvq_s32(self.0[1].0), vaddvq_s32(self.0[2].0), vaddvq_s32(self.0[3].0)] + }; + q[0].wrapping_add(q[1]) + .wrapping_add(q[2]) + .wrapping_add(q[3]) + } + + /// Minimum over all 16 lanes (`vminq_s32` tree, then `vminvq_s32`). + #[inline(always)] + pub fn reduce_min(self) -> i32 { + // SAFETY: NEON baseline; pure register ops. + unsafe { + let m = vminq_s32(vminq_s32(self.0[0].0, self.0[1].0), vminq_s32(self.0[2].0, self.0[3].0)); + vminvq_s32(m) + } + } + + /// Maximum over all 16 lanes (`vmaxq_s32` tree, then `vmaxvq_s32`). + #[inline(always)] + pub fn reduce_max(self) -> i32 { + // SAFETY: NEON baseline; pure register ops. + unsafe { + let m = vmaxq_s32(vmaxq_s32(self.0[0].0, self.0[1].0), vmaxq_s32(self.0[2].0, self.0[3].0)); + vmaxvq_s32(m) + } + } + + /// Lane-wise minimum (`vminq_s32`). + #[inline(always)] + pub fn simd_min(self, other: Self) -> Self { + Self(core::array::from_fn(|p| self.0[p].min(other.0[p]))) + } + + /// Lane-wise maximum (`vmaxq_s32`). + #[inline(always)] + pub fn simd_max(self, other: Self) -> Self { + Self(core::array::from_fn(|p| self.0[p].max(other.0[p]))) + } + + /// Lane-wise `i32 → f32` (`vcvtq_f32_s32` per quad) into the NEON `F32x16`. + #[inline(always)] + pub fn cast_f32(self) -> aarch64_simd::F32x16 { + // SAFETY: NEON baseline; pure register ops. + aarch64_simd::F32x16(core::array::from_fn(|p| unsafe { vcvtq_f32_s32(self.0[p].0) })) + } + + /// Lane-wise absolute value (`vabsq_s32`; `i32::MIN` wraps to itself, the + /// release-mode behaviour of the scalar backend). + #[inline(always)] + pub fn abs(self) -> Self { + // SAFETY: NEON baseline; pure register ops. + Self(core::array::from_fn(|p| I32x4(unsafe { vabsq_s32(self.0[p].0) }))) + } + + /// Sign-extend the first 16 `i16` of `s` (`vld1_s16` + `vmovl_s16` per quad). + /// Panics if `s.len() < 16`. + #[inline(always)] + pub fn from_i16_slice(s: &[i16]) -> Self { + assert!(s.len() >= 16); + // SAFETY: NEON baseline; `vld1_s16` reads 4 elements at an in-bounds + // offset (length asserted above). + Self(core::array::from_fn(|p| I32x4(unsafe { vmovl_s16(vld1_s16(s.as_ptr().add(4 * p))) }))) + } + + /// Truncate each lane to `i16` (`vmovn_s32` per quad; low 16 bits, like `as i16`). + #[inline(always)] + pub fn to_i16_array(self) -> [i16; 16] { + let mut o = [0i16; 16]; + for p in 0..4 { + // SAFETY: NEON baseline; store of 4 i16 at an in-bounds offset. + unsafe { vst1_s16(o.as_mut_ptr().add(4 * p), vmovn_s32(self.0[p].0)) }; + } + o + } + + /// Bit `i` set iff lane `i >= 0` (`vcgezq_s32` + vector pack), LSB-first. + #[inline(always)] + pub fn cmpge_zero_mask(self) -> u16 { + let mut m = 0u16; + for p in 0..4 { + // SAFETY: NEON baseline; pure register ops. + m |= quad_mask4(unsafe { vcgezq_s32(self.0[p].0) }) << (4 * p); + } + m + } + + /// Lane-wise **signed** greater-than as a packed 16-bit bitmask + /// (`vcgtq_s32` + vector pack). Bit `i` set iff `self.lane(i) > other.lane(i)`, + /// LSB-first; exact at `i32::MIN` / `i32::MAX`; signed, never bit-pattern. + /// Agrees bit-for-bit with the scalar correctness anchor. + #[inline(always)] + pub fn gt_bitmask(self, other: Self) -> u16 { + let mut m = 0u16; + for p in 0..4 { + // SAFETY: NEON baseline; pure register ops. + m |= quad_mask4(unsafe { vcgtq_s32(self.0[p].0, other.0[p].0) }) << (4 * p); + } + m + } +} + +#[cfg(target_arch = "aarch64")] +impl core::ops::Add for I32x16 { + type Output = Self; + #[inline(always)] + fn add(self, r: Self) -> Self { + Self(core::array::from_fn(|p| self.0[p].add(r.0[p]))) + } +} +#[cfg(target_arch = "aarch64")] +impl core::ops::Sub for I32x16 { + type Output = Self; + #[inline(always)] + fn sub(self, r: Self) -> Self { + Self(core::array::from_fn(|p| self.0[p].sub(r.0[p]))) + } +} +#[cfg(target_arch = "aarch64")] +impl core::ops::AddAssign for I32x16 { + #[inline(always)] + fn add_assign(&mut self, r: Self) { + *self = *self + r; + } +} +#[cfg(target_arch = "aarch64")] +impl core::ops::SubAssign for I32x16 { + #[inline(always)] + fn sub_assign(&mut self, r: Self) { + *self = *self - r; + } +} +#[cfg(target_arch = "aarch64")] +impl core::ops::Mul for I32x16 { + type Output = Self; + #[inline(always)] + fn mul(self, r: Self) -> Self { + // SAFETY: NEON baseline; `vmulq_s32` wraps, matching `wrapping_mul`. + Self(core::array::from_fn(|p| I32x4(unsafe { vmulq_s32(self.0[p].0, r.0[p].0) }))) + } +} +#[cfg(target_arch = "aarch64")] +impl core::ops::MulAssign for I32x16 { + #[inline(always)] + fn mul_assign(&mut self, r: Self) { + *self = *self * r; + } +} +#[cfg(target_arch = "aarch64")] +impl core::ops::Neg for I32x16 { + type Output = Self; + #[inline(always)] + fn neg(self) -> Self { + // SAFETY: NEON baseline; `vnegq_s32` wraps at i32::MIN (release semantics). + Self(core::array::from_fn(|p| I32x4(unsafe { vnegq_s32(self.0[p].0) }))) + } +} +#[cfg(target_arch = "aarch64")] +impl core::ops::BitAnd for I32x16 { + type Output = Self; + #[inline(always)] + fn bitand(self, r: Self) -> Self { + // SAFETY: NEON baseline; pure register ops. + Self(core::array::from_fn(|p| I32x4(unsafe { vandq_s32(self.0[p].0, r.0[p].0) }))) + } +} +#[cfg(target_arch = "aarch64")] +impl core::ops::BitOr for I32x16 { + type Output = Self; + #[inline(always)] + fn bitor(self, r: Self) -> Self { + // SAFETY: NEON baseline; pure register ops. + Self(core::array::from_fn(|p| I32x4(unsafe { vorrq_s32(self.0[p].0, r.0[p].0) }))) + } +} +#[cfg(target_arch = "aarch64")] +impl core::ops::BitXor for I32x16 { + type Output = Self; + #[inline(always)] + fn bitxor(self, r: Self) -> Self { + // SAFETY: NEON baseline; pure register ops. + Self(core::array::from_fn(|p| I32x4(unsafe { veorq_s32(self.0[p].0, r.0[p].0) }))) + } +} +#[cfg(target_arch = "aarch64")] +impl core::ops::BitAndAssign for I32x16 { + #[inline(always)] + fn bitand_assign(&mut self, r: Self) { + *self = *self & r; + } +} +#[cfg(target_arch = "aarch64")] +impl core::ops::BitOrAssign for I32x16 { + #[inline(always)] + fn bitor_assign(&mut self, r: Self) { + *self = *self | r; + } +} +#[cfg(target_arch = "aarch64")] +impl core::ops::BitXorAssign for I32x16 { + #[inline(always)] + fn bitxor_assign(&mut self, r: Self) { + *self = *self ^ r; + } +} +#[cfg(target_arch = "aarch64")] +impl core::ops::Not for I32x16 { + type Output = Self; + #[inline(always)] + fn not(self) -> Self { + // SAFETY: NEON baseline; pure register ops. + Self(core::array::from_fn(|p| I32x4(unsafe { vmvnq_s32(self.0[p].0) }))) + } +} +#[cfg(target_arch = "aarch64")] +impl PartialEq for I32x16 { + #[inline(always)] + fn eq(&self, other: &Self) -> bool { + // SAFETY: NEON baseline; `vceqq_s32` then an all-lanes check. + (0..4).all(|p| unsafe { vminvq_u32(vceqq_s32(self.0[p].0, other.0[p].0)) == u32::MAX }) + } +} +#[cfg(target_arch = "aarch64")] +impl core::fmt::Debug for I32x16 { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "I32x16({:?})", self.to_array()) + } +} + +// GEN-TERNLOG-BEGIN (tools/gen_ternlog_bodies.py — regenerate, do not hand-edit) +/// GENERATED by `tools/gen_ternlog_bodies.py` — a 2-input Boolean function +/// by its 4-bit table (bit `k` = value at index `(a << 1) | b`), at most +/// two operations. `#[inline]` (not `always`): the 256-table test would +/// otherwise carry every arm's temporaries in one debug frame. +#[cfg(target_arch = "aarch64")] +#[inline] +fn ternlog_two_input_u32x4(t: u8, a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { + // SAFETY: NEON is a baseline feature of every aarch64 target this module compiles + // for; these are pure register operations on values already in `uint32x4_t`. + unsafe { + match t & 0xF { + 0x0 => vdupq_n_u32(0), + 0x1 => vmvnq_u32(vorrq_u32(a, b)), + 0x2 => vandq_u32(vmvnq_u32(a), b), + 0x3 => vmvnq_u32(a), + 0x4 => vbicq_u32(a, b), + 0x5 => vmvnq_u32(b), + 0x6 => veorq_u32(a, b), + 0x7 => vmvnq_u32(vandq_u32(a, b)), + 0x8 => vandq_u32(a, b), + 0x9 => vmvnq_u32(veorq_u32(a, b)), + 0xa => b, + 0xb => vorrq_u32(vmvnq_u32(a), b), + 0xc => a, + 0xd => vorrq_u32(a, vmvnq_u32(b)), + 0xe => vorrq_u32(a, b), + _ => vdupq_n_u32(!0), + } + } +} + +/// GENERATED by `tools/gen_ternlog_bodies.py` — a 2-input Boolean function +/// by its 4-bit table (bit `k` = value at index `(a << 1) | b`), at most +/// two operations. `#[inline]` (not `always`): the 256-table test would +/// otherwise carry every arm's temporaries in one debug frame. +#[cfg(target_arch = "aarch64")] +#[inline] +fn ternlog_two_input_u64x2(t: u8, a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { + // SAFETY: NEON is a baseline feature of every aarch64 target this module compiles + // for; these are pure register operations on values already in `uint64x2_t`. + unsafe { + match t & 0xF { + 0x0 => vdupq_n_u64(0), + 0x1 => veorq_u64(vorrq_u64(a, b), vdupq_n_u64(!0)), + 0x2 => vandq_u64(veorq_u64(a, vdupq_n_u64(!0)), b), + 0x3 => veorq_u64(a, vdupq_n_u64(!0)), + 0x4 => vbicq_u64(a, b), + 0x5 => veorq_u64(b, vdupq_n_u64(!0)), + 0x6 => veorq_u64(a, b), + 0x7 => veorq_u64(vandq_u64(a, b), vdupq_n_u64(!0)), + 0x8 => vandq_u64(a, b), + 0x9 => veorq_u64(veorq_u64(a, b), vdupq_n_u64(!0)), + 0xa => b, + 0xb => vorrq_u64(veorq_u64(a, vdupq_n_u64(!0)), b), + 0xc => a, + 0xd => vorrq_u64(a, veorq_u64(b, vdupq_n_u64(!0))), + 0xe => vorrq_u64(a, b), + _ => vdupq_n_u64(!0), + } + } +} +// GEN-TERNLOG-END diff --git a/src/simd_nightly/f32_types.rs b/src/simd_nightly/f32_types.rs index 3370cf34..3bbf1c7c 100644 --- a/src/simd_nightly/f32_types.rs +++ b/src/simd_nightly/f32_types.rs @@ -7,6 +7,7 @@ use core::simd::{f32x16 as core_f32x16, f32x8 as core_f32x8}; // `mul_add`, `sqrt`, `round`, `floor`, `abs` live in `StdFloat` (std-only nightly trait). use std::simd::StdFloat; +use super::i_word_types::I32x16; use super::masks::{F32Mask16, F32Mask8}; use super::u_word_types::{U32x16, U32x8}; @@ -56,6 +57,48 @@ impl F32x16 { self.0.to_array() } + /// Gather 16 `f32` at `base_ptr.offset(indices[i])` — the `VPGATHERDD`- + /// shaped load the other backends expose under the same signature. + /// Portable SIMD has no gather over a raw pointer, so this reads lane by + /// lane. Indices are SIGNED element offsets, exactly as + /// `_mm512_i32gather_ps::<4>` treats them on the AVX-512 backend: a + /// negative index addresses an element before `base_ptr`, which is valid + /// whenever the caller's contract below holds (a first cut cast to + /// `usize`, turning `-1` into a huge positive offset — CodeRabbit on + /// PR #306). + /// + /// # Safety + /// + /// For every `i in 0..16`, `base_ptr.offset(indices[i] as isize)` must + /// lie inside one allocation together with `base_ptr`, be 4-byte aligned, + /// and point at an initialised, readable `f32`. + /// + /// # Examples + /// ```rust + /// # #[cfg(feature = "nightly-simd")] { + /// use ndarray::simd_nightly::{F32x16, I32x16}; + /// let table: Vec = (0..32).map(|i| i as f32).collect(); + /// // Index from the MIDDLE of the table so negative offsets are exercised. + /// let base = table[16..].as_ptr(); + /// let idx = I32x16::from_array([-16, -1, 0, 1, 15, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]); + /// // SAFETY: every offset lands inside `table`. + /// let g = unsafe { F32x16::gather(idx, base) }; + /// assert_eq!(g.to_array()[0], 0.0); + /// assert_eq!(g.to_array()[1], 15.0); + /// assert_eq!(g.to_array()[4], 31.0); + /// # } + /// ``` + #[inline(always)] + pub unsafe fn gather(indices: I32x16, base_ptr: *const f32) -> Self { + let idx = indices.to_array(); + let mut out = [0.0f32; 16]; + for (o, &i) in out.iter_mut().zip(idx.iter()) { + // SAFETY: validity of each address is the caller's contract (above). + *o = unsafe { *base_ptr.offset(i as isize) }; + } + Self::from_array(out) + } + /// Store all 16 lanes into the first 16 slots of `s`. /// /// # Panics diff --git a/src/simd_nightly/i8_types.rs b/src/simd_nightly/i8_types.rs index dde432e4..acd2e9b2 100644 --- a/src/simd_nightly/i8_types.rs +++ b/src/simd_nightly/i8_types.rs @@ -97,6 +97,56 @@ impl I8x64 { Self(self.0.simd_max(other.0)) } + /// Lane-wise minimum — the short name `simd_int_ops::min_i8` (the + /// integer facade, which still exists; the MASK family moved to + /// `simd_masking_ops`) calls on every backend; identical to + /// [`Self::simd_min`]. + /// + /// # Examples + /// ```rust + /// # #[cfg(feature = "nightly-simd")] { + /// use ndarray::simd_nightly::I8x64; + /// assert_eq!(I8x64::splat(-3).min(I8x64::splat(7)).to_array()[0], -3); + /// # } + /// ``` + #[inline(always)] + pub fn min(self, other: Self) -> Self { + Self(self.0.simd_min(other.0)) + } + + /// Lane-wise maximum — the short name `simd_int_ops::max_i8` calls on + /// every backend; identical to [`Self::simd_max`]. + /// + /// # Examples + /// ```rust + /// # #[cfg(feature = "nightly-simd")] { + /// use ndarray::simd_nightly::I8x64; + /// assert_eq!(I8x64::splat(-3).max(I8x64::splat(7)).to_array()[0], 7); + /// # } + /// ``` + #[inline(always)] + pub fn max(self, other: Self) -> Self { + Self(self.0.simd_max(other.0)) + } + + /// Saturating absolute value: `|i8::MIN|` is `i8::MAX` (127), never the + /// wrapped `i8::MIN` — the crate's `saturating_abs` contract (see the + /// VPABSB correction in `vertical-simd-consumer-contract.md`). + /// `core::simd`'s `saturating_abs` saturates identically. + /// + /// # Examples + /// ```rust + /// # #[cfg(feature = "nightly-simd")] { + /// use ndarray::simd_nightly::I8x64; + /// assert_eq!(I8x64::splat(i8::MIN).saturating_abs().to_array()[0], i8::MAX); + /// assert_eq!(I8x64::splat(-5).saturating_abs().to_array()[0], 5); + /// # } + /// ``` + #[inline(always)] + pub fn saturating_abs(self) -> Self { + Self(self.0.saturating_abs()) + } + // ── Saturating arithmetic ───────────────────────────────────── /// Per-lane signed saturating add. Results clamp to `[i8::MIN, i8::MAX]`. @@ -226,6 +276,56 @@ impl I8x32 { Self(self.0.simd_max(other.0)) } + /// Lane-wise minimum — the short name `simd_int_ops::min_i8` (the + /// integer facade, which still exists; the MASK family moved to + /// `simd_masking_ops`) calls on every backend; identical to + /// [`Self::simd_min`]. + /// + /// # Examples + /// ```rust + /// # #[cfg(feature = "nightly-simd")] { + /// use ndarray::simd_nightly::I8x32; + /// assert_eq!(I8x32::splat(-3).min(I8x32::splat(7)).to_array()[0], -3); + /// # } + /// ``` + #[inline(always)] + pub fn min(self, other: Self) -> Self { + Self(self.0.simd_min(other.0)) + } + + /// Lane-wise maximum — the short name `simd_int_ops::max_i8` calls on + /// every backend; identical to [`Self::simd_max`]. + /// + /// # Examples + /// ```rust + /// # #[cfg(feature = "nightly-simd")] { + /// use ndarray::simd_nightly::I8x32; + /// assert_eq!(I8x32::splat(-3).max(I8x32::splat(7)).to_array()[0], 7); + /// # } + /// ``` + #[inline(always)] + pub fn max(self, other: Self) -> Self { + Self(self.0.simd_max(other.0)) + } + + /// Saturating absolute value: `|i8::MIN|` is `i8::MAX` (127), never the + /// wrapped `i8::MIN` — the crate's `saturating_abs` contract (see the + /// VPABSB correction in `vertical-simd-consumer-contract.md`). + /// `core::simd`'s `saturating_abs` saturates identically. + /// + /// # Examples + /// ```rust + /// # #[cfg(feature = "nightly-simd")] { + /// use ndarray::simd_nightly::I8x32; + /// assert_eq!(I8x32::splat(i8::MIN).saturating_abs().to_array()[0], i8::MAX); + /// assert_eq!(I8x32::splat(-5).saturating_abs().to_array()[0], 5); + /// # } + /// ``` + #[inline(always)] + pub fn saturating_abs(self) -> Self { + Self(self.0.saturating_abs()) + } + // ── Saturating arithmetic ───────────────────────────────────── /// Per-lane signed saturating add. Results clamp to `[i8::MIN, i8::MAX]`. diff --git a/src/simd_nightly/i_word_types.rs b/src/simd_nightly/i_word_types.rs index ad3694ba..c42f3b3a 100644 --- a/src/simd_nightly/i_word_types.rs +++ b/src/simd_nightly/i_word_types.rs @@ -314,6 +314,98 @@ impl I32x16 { pub fn cmpgt_mask(self, other: Self) -> u16 { self.0.simd_gt(other.0).to_bitmask() as u16 } + + /// Per-lane signed greater-than as a packed 16-bit bitmask, LSB-first — + /// the name the agnostic mask surface (`simd_masking_ops`) calls on every + /// backend; identical to [`Self::cmpgt_mask`]. + /// + /// # Examples + /// ```rust + /// # #[cfg(feature = "nightly-simd")] { + /// use ndarray::simd_nightly::I32x16; + /// let mut a = [0i32; 16]; + /// a[0] = 1; + /// a[15] = i32::MAX; + /// a[7] = i32::MIN; + /// assert_eq!(I32x16::from_array(a).gt_bitmask(I32x16::splat(0)), 0b1000_0000_0000_0001); + /// # } + /// ``` + #[inline(always)] + pub fn gt_bitmask(self, other: Self) -> u16 { + self.0.simd_gt(other.0).to_bitmask() as u16 + } + + /// Load 16 × `i16` and sign-extend to 16 × `i32` (the `VPMOVSXWD` + /// widening the other backends perform; here `Simd::cast`). + /// + /// # Panics + /// + /// Panics if `s.len() < 16`. + /// + /// # Examples + /// ```rust + /// # #[cfg(feature = "nightly-simd")] { + /// use ndarray::simd_nightly::I32x16; + /// let s: [i16; 16] = core::array::from_fn(|i| (i as i16 - 8) * 1000); + /// let v = I32x16::from_i16_slice(&s); + /// assert_eq!(v.to_array()[0], -8000); + /// assert_eq!(v.to_i16_array(), s); + /// # } + /// ``` + #[inline(always)] + pub fn from_i16_slice(s: &[i16]) -> Self { + assert!(s.len() >= 16, "I32x16::from_i16_slice needs ≥16 elements"); + Self(i16x16::from_slice(s).cast::()) + } + + /// Narrow 16 × `i32` to 16 × `i16` by truncation (the `VPMOVDW` the + /// other backends perform; `Simd::cast` truncates like `as i16`). + /// + /// # Examples + /// ```rust + /// # #[cfg(feature = "nightly-simd")] { + /// use ndarray::simd_nightly::I32x16; + /// assert_eq!(I32x16::splat(0x1_0005).to_i16_array()[0], 5); // truncates like `as i16` + /// # } + /// ``` + #[inline(always)] + pub fn to_i16_array(self) -> [i16; 16] { + self.0.cast::().to_array() + } + + /// Lane-wise absolute value, WRAPPING at `i32::MIN` (`|i32::MIN|` stays + /// `i32::MIN`) — the same semantics as `VPABSD` on the other backends. + /// + /// # Examples + /// ```rust + /// # #[cfg(feature = "nightly-simd")] { + /// use ndarray::simd_nightly::I32x16; + /// assert_eq!(I32x16::splat(-9).abs().to_array()[0], 9); + /// assert_eq!(I32x16::splat(i32::MIN).abs().to_array()[0], i32::MIN); + /// # } + /// ``` + #[inline(always)] + pub fn abs(self) -> Self { + Self(self.0.abs()) + } + + /// Bit `i` set where lane `i >= 0` (LSB-first) — the sign-bit-clear test + /// the other backends expose under this name. + /// + /// # Examples + /// ```rust + /// # #[cfg(feature = "nightly-simd")] { + /// use ndarray::simd_nightly::I32x16; + /// let mut a = [-1i32; 16]; + /// a[0] = 0; + /// a[2] = 5; + /// assert_eq!(I32x16::from_array(a).cmpge_zero_mask(), 0b101); + /// # } + /// ``` + #[inline(always)] + pub fn cmpge_zero_mask(self) -> u16 { + self.0.simd_ge(i32x16::splat(0)).to_bitmask() as u16 + } } impl PartialEq for I32x16 { @@ -323,6 +415,23 @@ impl PartialEq for I32x16 { } } +/// Lane-wise WRAPPING multiply (low 32 bits of the product) — the `VPMULLD` +/// semantics of the other backends; `core::simd`'s `*` wraps identically. +impl core::ops::Mul for I32x16 { + type Output = Self; + #[inline(always)] + fn mul(self, rhs: Self) -> Self { + Self(self.0 * rhs.0) + } +} + +impl core::ops::MulAssign for I32x16 { + #[inline(always)] + fn mul_assign(&mut self, rhs: Self) { + self.0 *= rhs.0; + } +} + impl core::fmt::Display for I32x16 { fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { write!(f, "I32x16({:?})", &self.to_array()[..]) diff --git a/src/simd_nightly/mod.rs b/src/simd_nightly/mod.rs index 3a281acf..3e0bd832 100644 --- a/src/simd_nightly/mod.rs +++ b/src/simd_nightly/mod.rs @@ -29,6 +29,7 @@ pub mod masks; pub mod ops; pub mod u8_types; pub mod u_word_types; +pub mod w1a_types; #[cfg(test)] mod tests; @@ -43,6 +44,9 @@ pub use i_word_types::{I16x16, I16x32, I32x16, I32x8, I64x4, I64x8}; pub use masks::{F32Mask16, F32Mask8, F64Mask4, F64Mask8}; pub use u8_types::{U8x32, U8x64}; pub use u_word_types::{U16x16, U16x32, U32x16, U32x8, U64x4, U64x8}; +pub use w1a_types::{ + batch_packed_i4_16, palette_lookup_u8x8, prefetch_read_t0, prefetch_read_t1, prefetch_read_t2, I8x16, U16x8, U8x8, +}; // Lowercase aliases — match the std::simd convention used by // `simd_avx2.rs`, `simd_avx512.rs`, and the scalar fallback in @@ -90,3 +94,11 @@ pub type u16x16 = U16x16; pub type i32x8 = I32x8; #[allow(non_camel_case_types)] pub type i64x4 = I64x4; +// W1a consumer-contract lanes (2026-09-14): the same three aliases every +// other backend file exports. +#[allow(non_camel_case_types)] +pub type i8x16 = I8x16; +#[allow(non_camel_case_types)] +pub type u16x8 = U16x8; +#[allow(non_camel_case_types)] +pub type u8x8 = U8x8; diff --git a/src/simd_nightly/ops.rs b/src/simd_nightly/ops.rs index 6a5cdb5d..714b0824 100644 --- a/src/simd_nightly/ops.rs +++ b/src/simd_nightly/ops.rs @@ -2,7 +2,9 @@ //! //! Two main macros: //! - `impl_fp_ops!` — Add/Sub/Mul/Div/Neg + assign variants for float types. -//! - `impl_int_ops!` — Add/Sub/BitAnd/BitOr/BitXor + assign variants for int types. +//! - `impl_int_ops!` — Add/Sub/BitAnd/BitOr/BitXor/Not + assign variants for int types +//! (`Not` added 2026-09-14: the mask family's `!U64x8` complement did not compile on +//! this backend — the first of three gaps the five-flavour audit of #306 closed here). //! //! A separate `impl_default!` macro handles `Default` for types whose own //! module has NOT already derived/implemented it (e.g. F32x16/F32x8 already @@ -170,6 +172,13 @@ macro_rules! impl_int_ops { } } + impl core::ops::Not for $name { + type Output = Self; + #[inline(always)] + fn not(self) -> Self { + Self(!self.0) + } + } impl core::ops::BitXorAssign for $name { #[inline(always)] fn bitxor_assign(&mut self, rhs: Self) { diff --git a/src/simd_nightly/u_word_types.rs b/src/simd_nightly/u_word_types.rs index 00f03b6e..71e9bd33 100644 --- a/src/simd_nightly/u_word_types.rs +++ b/src/simd_nightly/u_word_types.rs @@ -3,6 +3,7 @@ use core::simd::cmp::{SimdOrd, SimdPartialEq, SimdPartialOrd}; use core::simd::num::SimdUint; +use core::simd::simd_swizzle; use core::simd::{u16x16, u16x32, u32x16, u32x8, u64x4, u64x8}; // ════════════════════════════════════════════════════════════════════ @@ -130,6 +131,38 @@ impl U64x8 { pub fn cmpgt_mask(self, other: Self) -> u8 { self.0.simd_gt(other.0).to_bitmask() as u8 } + + /// Lane-wise population count (`u64::count_ones` per lane, as `u64`). + /// + /// # Examples + /// ```rust + /// # #[cfg(feature = "nightly-simd")] { + /// use ndarray::simd_nightly::U64x8; + /// let v = U64x8::from_array([u64::MAX, 0, 1, !1, 3, 0xF0, 1 << 63, 0xFFFF]); + /// assert_eq!(v.popcnt().to_array(), [64, 0, 1, 63, 2, 4, 1, 16]); + /// # } + /// ``` + #[inline(always)] + pub fn popcnt(self) -> Self { + Self(self.0.count_ones()) + } + + /// `popcount(self ^ other)` summed over all 8 lanes — the Hamming distance + /// of two 512-bit fingerprints. + /// + /// # Examples + /// ```rust + /// # #[cfg(feature = "nightly-simd")] { + /// use ndarray::simd_nightly::U64x8; + /// let a = U64x8::splat(u64::MAX); + /// assert_eq!(a.xor_popcount(U64x8::splat(0)), 512); + /// assert_eq!(a.xor_popcount(a), 0); + /// # } + /// ``` + #[inline(always)] + pub fn xor_popcount(self, other: Self) -> u64 { + (self.0 ^ other.0).count_ones().reduce_sum() + } } impl Default for U64x8 { @@ -222,6 +255,20 @@ impl U64x4 { pub fn cmpgt_mask(self, other: Self) -> u8 { self.0.simd_gt(other.0).to_bitmask() as u8 } + + /// Lane-wise population count (`u64::count_ones` per lane, as `u64`). + /// + /// # Examples + /// ```rust + /// # #[cfg(feature = "nightly-simd")] { + /// use ndarray::simd_nightly::U64x4; + /// assert_eq!(U64x4::from_array([u64::MAX, 0, 1, !1]).popcnt().to_array(), [64, 0, 1, 63]); + /// # } + /// ``` + #[inline(always)] + pub fn popcnt(self) -> Self { + Self(self.0.count_ones()) + } } impl Default for U64x4 { @@ -314,6 +361,96 @@ impl U32x8 { pub fn cmpgt_mask(self, other: Self) -> u8 { self.0.simd_gt(other.0).to_bitmask() as u8 } + + /// Lane-wise left-rotate by `n` bits (`n` mod 32; `0` returns `self`) — + /// the 8-lane twin of `U32x16::rotate_left`. + /// + /// # Examples + /// ```rust + /// # #[cfg(feature = "nightly-simd")] { + /// use ndarray::simd_nightly::U32x8; + /// let v = U32x8::splat(0x8000_0001); + /// assert_eq!(v.rotate_left(1).to_array()[0], 0x0000_0003); + /// assert_eq!(v.rotate_left(32).to_array()[0], 0x8000_0001); + /// # } + /// ``` + #[inline(always)] + pub fn rotate_left(self, n: u32) -> Self { + let n = n % 32; + if n == 0 { + return self; + } + Self((self.0 << u32x8::splat(n)) | (self.0 >> u32x8::splat(32 - n))) + } + + /// `_mm256_unpacklo_epi32`: `[a0,b0,a1,b1, a4,b4,a5,b5]` — the per-128-bit-lane + /// interleave every backend exposes under this name (BLAKE3's transpose + /// vocabulary); a compile-time `simd_swizzle!` here. + /// + /// # Examples + /// The six lane shuffles, on `a = [0..8)` and `b = [10..18)`: + /// ```rust + /// # #[cfg(feature = "nightly-simd")] { + /// use ndarray::simd_nightly::U32x8; + /// let a = U32x8::from_array([0, 1, 2, 3, 4, 5, 6, 7]); + /// let b = U32x8::from_array([10, 11, 12, 13, 14, 15, 16, 17]); + /// assert_eq!(a.interleave_lo_u32(b).to_array(), [0, 10, 1, 11, 4, 14, 5, 15]); + /// assert_eq!(a.interleave_hi_u32(b).to_array(), [2, 12, 3, 13, 6, 16, 7, 17]); + /// assert_eq!(a.interleave_lo_u64(b).to_array(), [0, 1, 10, 11, 4, 5, 14, 15]); + /// assert_eq!(a.interleave_hi_u64(b).to_array(), [2, 3, 12, 13, 6, 7, 16, 17]); + /// assert_eq!(a.concat_lo_halves(b).to_array(), [0, 1, 2, 3, 10, 11, 12, 13]); + /// assert_eq!(a.concat_hi_halves(b).to_array(), [4, 5, 6, 7, 14, 15, 16, 17]); + /// # } + /// ``` + #[inline(always)] + pub fn interleave_lo_u32(self, other: Self) -> Self { + Self(simd_swizzle!(self.0, other.0, [0, 8, 1, 9, 4, 12, 5, 13])) + } + + /// `_mm256_unpackhi_epi32`: `[a2,b2,a3,b3, a6,b6,a7,b7]`. + /// + /// # Examples + /// See [`Self::interleave_lo_u32`] — one worked example covers all six shuffles. + #[inline(always)] + pub fn interleave_hi_u32(self, other: Self) -> Self { + Self(simd_swizzle!(self.0, other.0, [2, 10, 3, 11, 6, 14, 7, 15])) + } + + /// `_mm256_unpacklo_epi64`: `[a0,a1,b0,b1, a4,a5,b4,b5]`. + /// + /// # Examples + /// See [`Self::interleave_lo_u32`] — one worked example covers all six shuffles. + #[inline(always)] + pub fn interleave_lo_u64(self, other: Self) -> Self { + Self(simd_swizzle!(self.0, other.0, [0, 1, 8, 9, 4, 5, 12, 13])) + } + + /// `_mm256_unpackhi_epi64`: `[a2,a3,b2,b3, a6,a7,b6,b7]`. + /// + /// # Examples + /// See [`Self::interleave_lo_u32`] — one worked example covers all six shuffles. + #[inline(always)] + pub fn interleave_hi_u64(self, other: Self) -> Self { + Self(simd_swizzle!(self.0, other.0, [2, 3, 10, 11, 6, 7, 14, 15])) + } + + /// `_mm256_permute2x128_si256(a, b, 0x20)`: `[a0..a3, b0..b3]`. + /// + /// # Examples + /// See [`Self::interleave_lo_u32`] — one worked example covers all six shuffles. + #[inline(always)] + pub fn concat_lo_halves(self, other: Self) -> Self { + Self(simd_swizzle!(self.0, other.0, [0, 1, 2, 3, 8, 9, 10, 11])) + } + + /// `_mm256_permute2x128_si256(a, b, 0x31)`: `[a4..a7, b4..b7]`. + /// + /// # Examples + /// See [`Self::interleave_lo_u32`] — one worked example covers all six shuffles. + #[inline(always)] + pub fn concat_hi_halves(self, other: Self) -> Self { + Self(simd_swizzle!(self.0, other.0, [4, 5, 6, 7, 12, 13, 14, 15])) + } } impl Default for U32x8 { @@ -566,6 +703,25 @@ impl U32x16 { self.0.simd_gt(other.0).to_bitmask() as u16 } + /// Per-lane equality as a packed 16-bit bitmask, LSB-first — the name the + /// agnostic mask surface (`simd_masking_ops`) calls on every backend + /// (`cmpeq_mask` is this backend's older spelling of the same thing). + /// + /// # Examples + /// ```rust + /// # #[cfg(feature = "nightly-simd")] { + /// use ndarray::simd_nightly::U32x16; + /// let mut a = [0u32; 16]; + /// a[0] = 7; + /// a[15] = 7; + /// assert_eq!(U32x16::from_array(a).eq_bitmask(U32x16::splat(7)), 0b1000_0000_0000_0001); + /// # } + /// ``` + #[inline(always)] + pub fn eq_bitmask(self, other: Self) -> u16 { + self.0.simd_eq(other.0).to_bitmask() as u16 + } + /// Lane-wise left-rotate by `n` bits — the ARX rotate (matches /// `u32::rotate_left`), completing `Add` + `BitXor` for ChaCha20/BLAKE. /// `core::simd` has no bit-rotate, so this is the shift-or composition with diff --git a/src/simd_nightly/w1a_types.rs b/src/simd_nightly/w1a_types.rs new file mode 100644 index 00000000..0bd44c95 --- /dev/null +++ b/src/simd_nightly/w1a_types.rs @@ -0,0 +1,561 @@ +//! W1a consumer-contract primitives on the portable-simd backend. +//! +//! `I8x16` / `U16x8` / `U8x8` / `palette_lookup_u8x8` / `prefetch_read_t*` / +//! `batch_packed_i4_16` exist on every intrinsics backend (`simd_avx512`, +//! `simd_avx2`, `simd_neon`, `simd_wasm`, `simd_scalar`); until 2026-09-14 the +//! `nightly-simd` realization had none of them, so a consumer that compiled +//! against `crate::simd::I8x16` broke the moment the feature was turned on. +//! The polyfill law is that every backend file realizes the whole facade, so +//! they land here as thin `core::simd` bodies — never as a `#[cfg]` that hides +//! the facade tests under this realization. +//! +//! Semantics are the scalar backend's, bit for bit: `from_i4_packed_u64` +//! sign-extends nibble `0x8` to `-8`, `saturating_abs(i8::MIN) == i8::MAX` +//! (the VPABSB correction in `vertical-simd-consumer-contract.md`), the +//! gathers are bounds-checked in debug and return `0` for an out-of-range +//! index in release, and the prefetches are documented no-ops (a hint has no +//! observable result, and `core::simd` carries no prefetch). +#![cfg(feature = "nightly-simd")] + +use core::fmt; +use core::simd::cmp::{SimdOrd, SimdPartialEq, SimdPartialOrd}; +use core::simd::num::{SimdInt, SimdUint}; +use core::simd::{i8x16 as core_i8x16, u16x8 as core_u16x8, u64x16, u8x8 as core_u8x8, Simd}; + +// ── W1a-#1: I8x16 + lane_i8 + from_i4_packed_u64 ──────────────────────────── + +/// 16-lane `i8` vector backed by `core::simd::i8x16`. +/// +/// Mirrors `simd_scalar::I8x16` / `simd_neon::I8x16` so consumer code is +/// backend-agnostic; every method executes under miri. +/// +/// # Examples +/// ```rust +/// # #[cfg(feature = "nightly-simd")] { +/// use ndarray::simd_nightly::I8x16; +/// let v = I8x16::from_i4_packed_u64(0x8); +/// assert_eq!(v.lane_i8::<0>(), -8); +/// assert_eq!(v.lane_i8::<1>(), 0); +/// # } +/// ``` +#[derive(Copy, Clone)] +#[repr(transparent)] +pub struct I8x16(pub core_i8x16); + +impl I8x16 { + /// Number of `i8` lanes. + pub const LANES: usize = 16; + + /// Broadcast a single `i8` value to all 16 lanes. + #[inline(always)] + pub fn splat(v: i8) -> Self { + Self(core_i8x16::splat(v)) + } + + /// Load from a slice (at least 16 elements required; panics otherwise). + #[inline(always)] + pub fn from_slice(s: &[i8]) -> Self { + assert!(s.len() >= 16); + Self(core_i8x16::from_slice(s)) + } + + /// Load from a fixed-size array. + #[inline(always)] + pub fn from_array(arr: [i8; 16]) -> Self { + Self(core_i8x16::from_array(arr)) + } + + /// Extract all 16 lanes as an array. + #[inline(always)] + pub fn to_array(self) -> [i8; 16] { + self.0.to_array() + } + + /// Copy lanes into a slice (must have at least 16 elements). + #[inline(always)] + pub fn copy_to_slice(self, s: &mut [i8]) { + assert!(s.len() >= 16); + self.0.copy_to_slice(&mut s[..16]); + } + + /// Unpack 16 signed i4 nibbles from a `u64` into 16 sign-extended `i8` + /// lanes: `lane[i] = sign_extend_i4((packed >> (4*i)) & 0xf)`, so + /// `0x0..=0x7 → 0..=7` and `0x8..=0xf → -8..=-1`. + /// + /// The sign extension is the `(x << 4) >> 4` arithmetic-shift identity on + /// the i8 lane, which is exactly what the scalar backend's + /// `if nibble > 7 { nibble - 16 }` computes. + /// + /// # Examples + /// ```rust + /// # #[cfg(feature = "nightly-simd")] { + /// use ndarray::simd_nightly::I8x16; + /// assert_eq!(I8x16::from_i4_packed_u64(0).to_array(), [0i8; 16]); + /// assert_eq!(I8x16::from_i4_packed_u64(u64::MAX).to_array(), [-1i8; 16]); + /// let v = I8x16::from_i4_packed_u64(0x8_7); + /// assert_eq!((v.lane_i8::<0>(), v.lane_i8::<1>()), (7, -8)); + /// # } + /// ``` + #[inline(always)] + pub fn from_i4_packed_u64(packed: u64) -> Self { + const SHIFTS: u64x16 = Simd::from_array([0, 4, 8, 12, 16, 20, 24, 28, 32, 36, 40, 44, 48, 52, 56, 60]); + let nibbles = (u64x16::splat(packed) >> SHIFTS) & u64x16::splat(0xf); + let lanes = nibbles.cast::(); + Self((lanes << core_i8x16::splat(4)) >> core_i8x16::splat(4)) + } + + /// Extract lane `N` as an `i8`. `N` must be in `0..16`. + /// + /// # Examples + /// ```rust + /// # #[cfg(feature = "nightly-simd")] { + /// use ndarray::simd_nightly::I8x16; + /// let v = I8x16::from_array(core::array::from_fn(|i| i as i8 * 3)); + /// assert_eq!(v.lane_i8::<5>(), 15); + /// # } + /// ``` + #[inline(always)] + pub fn lane_i8(self) -> i8 { + self.0[N] + } + + /// Lane-wise saturating absolute value: `saturating_abs(i8::MIN) == i8::MAX`. + /// + /// # Examples + /// ```rust + /// # #[cfg(feature = "nightly-simd")] { + /// use ndarray::simd_nightly::I8x16; + /// assert_eq!(I8x16::splat(i8::MIN).saturating_abs().to_array(), [i8::MAX; 16]); + /// assert_eq!(I8x16::splat(-4).saturating_abs().to_array(), [4; 16]); + /// # } + /// ``` + #[inline(always)] + pub fn saturating_abs(self) -> Self { + Self(self.0.saturating_abs()) + } + + /// Lane-wise minimum. + /// + /// # Examples + /// See [`Self::cmpeq_mask`] — one example exercises the four compare / min / max methods. + #[inline(always)] + pub fn simd_min(self, other: Self) -> Self { + Self(self.0.simd_min(other.0)) + } + + /// Lane-wise maximum. + /// + /// # Examples + /// See [`Self::cmpeq_mask`] — one example exercises the four compare / min / max methods. + #[inline(always)] + pub fn simd_max(self, other: Self) -> Self { + Self(self.0.simd_max(other.0)) + } + + /// Per-lane `self == other`, as a 16-bit mask (bit `i` = lane `i`). + /// + /// # Examples + /// ```rust + /// # #[cfg(feature = "nightly-simd")] { + /// use ndarray::simd_nightly::I8x16; + /// let mut a = [0i8; 16]; + /// a[0] = 9; + /// a[15] = 9; + /// let v = I8x16::from_array(a); + /// assert_eq!(v.cmpeq_mask(I8x16::splat(9)), 0b1000_0000_0000_0001); + /// assert_eq!(v.cmpgt_mask(I8x16::splat(0)), 0b1000_0000_0000_0001); + /// assert_eq!(v.simd_min(I8x16::splat(3)).to_array()[0], 3); + /// assert_eq!(v.simd_max(I8x16::splat(3)).to_array()[1], 3); + /// # } + /// ``` + #[inline(always)] + pub fn cmpeq_mask(self, other: Self) -> u16 { + self.0.simd_eq(other.0).to_bitmask() as u16 + } + + /// Per-lane signed `self > other`, as a 16-bit mask. + /// + /// # Examples + /// See [`Self::cmpeq_mask`] — one example exercises the four compare / min / max methods. + #[inline(always)] + pub fn cmpgt_mask(self, other: Self) -> u16 { + self.0.simd_gt(other.0).to_bitmask() as u16 + } +} + +impl PartialEq for I8x16 { + fn eq(&self, other: &Self) -> bool { + self.to_array() == other.to_array() + } +} + +impl fmt::Debug for I8x16 { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "I8x16({:?})", &self.to_array()[..]) + } +} + +// ── W1a-#3: U16x8 / U8x8 / palette_lookup_u8x8 ───────────────────────────── + +/// 8-lane `u16` vector backed by `core::simd::u16x8`. +/// +/// # Examples +/// ```rust +/// # #[cfg(feature = "nightly-simd")] { +/// use ndarray::simd_nightly::U16x8; +/// let table = [10u16, 20, 30, 40, 50, 60, 70, 80]; +/// let idx = U16x8::from_array([0, 2, 4, 6, 1, 3, 5, 7]); +/// assert_eq!(U16x8::gather_u16(idx, &table).to_array(), [10, 30, 50, 70, 20, 40, 60, 80]); +/// # } +/// ``` +#[derive(Copy, Clone)] +#[repr(transparent)] +pub struct U16x8(pub core_u16x8); + +impl U16x8 { + /// Number of `u16` lanes. + pub const LANES: usize = 8; + + /// Broadcast a single `u16` to all 8 lanes. + #[inline(always)] + pub fn splat(v: u16) -> Self { + Self(core_u16x8::splat(v)) + } + + /// Load from a slice (at least 8 elements required; panics otherwise). + #[inline(always)] + pub fn from_slice(s: &[u16]) -> Self { + assert!(s.len() >= 8); + Self(core_u16x8::from_slice(s)) + } + + /// Load from a fixed-size array. + #[inline(always)] + pub fn from_array(arr: [u16; 8]) -> Self { + Self(core_u16x8::from_array(arr)) + } + + /// Extract all 8 lanes as an array. + #[inline(always)] + pub fn to_array(self) -> [u16; 8] { + self.0.to_array() + } + + /// Gather 8 `u16` values from `table` at the indices in `indices`. + /// + /// Panics in debug if any index is `>= table.len()`; in release an + /// out-of-range index yields `0` (the scalar backend's rule, kept so the + /// two realizations never disagree on the same input). + /// + /// # Examples + /// ```rust + /// # #[cfg(feature = "nightly-simd")] { + /// use ndarray::simd_nightly::U16x8; + /// let table = [10u16, 20, 30, 40, 50, 60, 70, 80]; + /// let idx = U16x8::from_array([0, 2, 4, 6, 1, 3, 5, 7]); + /// assert_eq!(U16x8::gather_u16(idx, &table).to_array(), [10, 30, 50, 70, 20, 40, 60, 80]); + /// assert_eq!(U16x8::gather_u16(idx, &table).lane(7), 80); + /// # } + /// ``` + #[inline(always)] + pub fn gather_u16(indices: U16x8, table: &[u16]) -> Self { + let idx = indices.to_array(); + #[cfg(debug_assertions)] + for &i in &idx { + assert!((i as usize) < table.len(), "gather_u16: index {} OOB (len={})", i, table.len()); + } + let mut out = [0u16; 8]; + for k in 0..8 { + out[k] = table.get(idx[k] as usize).copied().unwrap_or(0); + } + Self::from_array(out) + } + + /// Extract lane `k` as a `u16`. + #[inline(always)] + pub fn lane(self, k: usize) -> u16 { + self.0[k] + } + + /// Lane-wise minimum. + #[inline(always)] + pub fn simd_min(self, other: Self) -> Self { + Self(self.0.simd_min(other.0)) + } + + /// Lane-wise maximum. + #[inline(always)] + pub fn simd_max(self, other: Self) -> Self { + Self(self.0.simd_max(other.0)) + } + + /// Horizontal wrapping sum of all 8 lanes. + /// + /// # Examples + /// ```rust + /// # #[cfg(feature = "nightly-simd")] { + /// use ndarray::simd_nightly::U16x8; + /// let v = U16x8::from_array([1, 2, 3, 4, 5, 6, 7, 8]); + /// assert_eq!(v.reduce_sum(), 36); + /// assert_eq!(v.simd_min(U16x8::splat(4)).to_array(), [1, 2, 3, 4, 4, 4, 4, 4]); + /// assert_eq!(v.simd_max(U16x8::splat(4)).to_array(), [4, 4, 4, 4, 5, 6, 7, 8]); + /// # } + /// ``` + #[inline(always)] + pub fn reduce_sum(self) -> u16 { + self.0.reduce_sum() + } +} + +impl PartialEq for U16x8 { + fn eq(&self, other: &Self) -> bool { + self.to_array() == other.to_array() + } +} + +impl fmt::Debug for U16x8 { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "U16x8({:?})", &self.to_array()[..]) + } +} + +/// 8-lane `u8` vector backed by `core::simd::u8x8` — the return type of +/// [`palette_lookup_u8x8`]. +#[derive(Copy, Clone)] +#[repr(transparent)] +pub struct U8x8(pub core_u8x8); + +impl U8x8 { + /// Number of `u8` lanes. + pub const LANES: usize = 8; + + /// Broadcast a single `u8` to all 8 lanes. + #[inline(always)] + pub fn splat(v: u8) -> Self { + Self(core_u8x8::splat(v)) + } + + /// Load from a fixed-size array. + #[inline(always)] + pub fn from_array(arr: [u8; 8]) -> Self { + Self(core_u8x8::from_array(arr)) + } + + /// Extract all 8 lanes as an array. + #[inline(always)] + pub fn to_array(self) -> [u8; 8] { + self.0.to_array() + } + + /// Horizontal wrapping sum of all 8 lanes. + /// + /// # Examples + /// ```rust + /// # #[cfg(feature = "nightly-simd")] { + /// use ndarray::simd_nightly::U8x8; + /// assert_eq!(U8x8::from_array([1, 2, 3, 4, 5, 6, 7, 8]).reduce_sum(), 36); + /// assert_eq!(U8x8::splat(255).reduce_sum(), 255u8.wrapping_mul(8)); + /// # } + /// ``` + #[inline(always)] + pub fn reduce_sum(self) -> u8 { + self.0.reduce_sum() + } +} + +impl PartialEq for U8x8 { + fn eq(&self, other: &Self) -> bool { + self.to_array() == other.to_array() + } +} + +impl fmt::Debug for U8x8 { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "U8x8({:?})", &self.to_array()[..]) + } +} + +/// Look up 8 bytes from a `u8` LUT by `u16` indices. +/// +/// Panics in debug on an out-of-range index; returns `0` for it in release — +/// identical to the scalar backend. +/// +/// # Examples +/// ```rust +/// # #[cfg(feature = "nightly-simd")] { +/// use ndarray::simd_nightly::{palette_lookup_u8x8, U16x8}; +/// let lut: Vec = (0..=255u8).rev().collect(); +/// let idx = U16x8::from_array([0, 1, 2, 255, 100, 200, 3, 4]); +/// assert_eq!(palette_lookup_u8x8(idx, &lut).to_array(), [255, 254, 253, 0, 155, 55, 252, 251]); +/// # } +/// ``` +#[inline(always)] +pub fn palette_lookup_u8x8(idx_v: U16x8, lut: &[u8]) -> U8x8 { + let idx = idx_v.to_array(); + #[cfg(debug_assertions)] + for &i in &idx { + assert!((i as usize) < lut.len(), "palette_lookup_u8x8: index {} OOB (len={})", i, lut.len()); + } + let mut out = [0u8; 8]; + for k in 0..8 { + out[k] = lut.get(idx[k] as usize).copied().unwrap_or(0); + } + U8x8::from_array(out) +} + +// ── W1a-#4: prefetch_read_t0/t1/t2 ────────────────────────────────────────── + +/// Hint that `ptr` will be read soon. A deliberate no-op on this backend: +/// `core::simd` carries no prefetch, and the contract is a hint with no +/// observable result. `ptr` may be invalid; it is never dereferenced. +/// +/// # Examples +/// ```rust +/// # #[cfg(feature = "nightly-simd")] { +/// use ndarray::simd_nightly::{prefetch_read_t0, prefetch_read_t1, prefetch_read_t2}; +/// let buf = [0u8; 64]; +/// prefetch_read_t0(buf.as_ptr()); +/// prefetch_read_t1(core::ptr::null()); // a hint never dereferences +/// prefetch_read_t2(buf.as_ptr()); +/// # } +/// ``` +#[inline(always)] +pub fn prefetch_read_t0(_ptr: *const u8) {} + +/// Hint to load into L2 (T1) cache — no-op on this backend, see +/// [`prefetch_read_t0`]. +#[inline(always)] +pub fn prefetch_read_t1(_ptr: *const u8) {} + +/// Hint to load into L3 (T2) cache — no-op on this backend, see +/// [`prefetch_read_t0`]. +#[inline(always)] +pub fn prefetch_read_t2(_ptr: *const u8) {} + +// ── W1a-#1: batch_packed_i4_16 ────────────────────────────────────────────── + +/// Closure-parameterised batch over packed i4 data. +/// +/// Iterates `min(packed.len(), out.len())` times; each iteration unpacks +/// `packed[i]` into an [`I8x16`] and passes it with `aux[i]` to `f`, storing +/// the result in `out[i]`. Panics if `packed.len() != aux.len()`. +/// +/// # Examples +/// ```rust +/// # #[cfg(feature = "nightly-simd")] { +/// use ndarray::simd_nightly::batch_packed_i4_16; +/// let packed = [0x7777_7777_7777_7777u64, 0x8888_8888_8888_8888]; +/// let aux = [1i8, 2]; +/// let mut out = [0i32; 2]; +/// batch_packed_i4_16(&packed, &aux, &mut out, |v, a| { +/// v.to_array().iter().map(|&x| x as i32).sum::() * a as i32 +/// }); +/// assert_eq!(out, [16 * 7, 16 * -8 * 2]); +/// # } +/// ``` +#[inline] +pub fn batch_packed_i4_16(packed: &[u64], aux: &[i8], out: &mut [E], f: F) +where + F: Fn(I8x16, i8) -> E + Sync + Send, + E: Copy, +{ + assert_eq!(packed.len(), aux.len(), "batch_packed_i4_16: packed and aux must be same length"); + let n = packed.len().min(out.len()); + for i in 0..n { + out[i] = f(I8x16::from_i4_packed_u64(packed[i]), aux[i]); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The scalar backend's rule, restated independently of `core::simd` so + /// the portable body is checked against something that is not itself. + fn reference_unpack(packed: u64) -> [i8; 16] { + let mut lanes = [0i8; 16]; + for (i, lane) in lanes.iter_mut().enumerate() { + let nibble = ((packed >> (4 * i)) & 0xf) as i8; + *lane = if nibble > 7 { nibble - 16 } else { nibble }; + } + lanes + } + + #[test] + fn i4_unpack_sign_extends_every_nibble_value_in_every_lane() { + // Every nibble value at every lane position, plus the two all-same + // words. A body that forgot the arithmetic shift passes `0..=7` and + // fails `0x8..=0xf`; one that sign-extended the wrong width fails the + // per-lane placement. + for lane in 0..16 { + for nib in 0u64..16 { + let packed = nib << (4 * lane); + assert_eq!( + I8x16::from_i4_packed_u64(packed).to_array(), + reference_unpack(packed), + "lane {lane} nibble {nib:#x}" + ); + } + } + assert_eq!(I8x16::from_i4_packed_u64(u64::MAX).to_array(), [-1i8; 16]); + assert_eq!(I8x16::from_i4_packed_u64(0x8888_8888_8888_8888).to_array(), [-8i8; 16]); + let mixed = 0xfedc_ba98_7654_3210u64; + let got = I8x16::from_i4_packed_u64(mixed); + assert_eq!(got.to_array(), reference_unpack(mixed)); + assert_eq!(got.lane_i8::<0>(), 0); + assert_eq!(got.lane_i8::<7>(), 7); + assert_eq!(got.lane_i8::<8>(), -8); + assert_eq!(got.lane_i8::<15>(), -1); + } + + #[test] + fn saturating_abs_saturates_i8_min_and_leaves_the_rest_exact() { + let mut arr = [0i8; 16]; + arr[0] = i8::MIN; + arr[1] = -127; + arr[2] = -1; + arr[3] = 0; + arr[4] = 1; + arr[5] = i8::MAX; + let got = I8x16::from_array(arr).saturating_abs().to_array(); + assert_eq!(got[0], i8::MAX, "|i8::MIN| must saturate to 127, not wrap to -128"); + assert_eq!(&got[1..6], &[127, 1, 0, 1, 127]); + } + + #[test] + fn gather_and_palette_lookup_index_by_lane() { + let table: Vec = (0..300).map(|i| i as u16 * 3).collect(); + let idx = U16x8::from_array([0, 299, 1, 298, 2, 297, 3, 296]); + assert_eq!(U16x8::gather_u16(idx, &table).to_array(), [0, 897, 3, 894, 6, 891, 9, 888]); + let lut: Vec = (0..=255u8).rev().collect(); + assert_eq!( + palette_lookup_u8x8(idx.simd_min(U16x8::splat(255)), &lut).to_array(), + [255, 0, 254, 0, 253, 0, 252, 0] + ); + } + + #[test] + fn batch_unpacks_each_word_and_stops_at_the_shorter_output() { + let packed = [0x0u64, u64::MAX, 0x8888_8888_8888_8888, 0x7777_7777_7777_7777]; + let aux = [1i8, 2, 3, 4]; + let mut out = [i32::MIN; 3]; + batch_packed_i4_16(&packed, &aux, &mut out, |v, a| { + v.to_array().iter().map(|&x| x as i32).sum::() * a as i32 + }); + assert_eq!(out, [0, -16 * 2, -128 * 3]); + } + + #[test] + #[should_panic(expected = "same length")] + fn batch_rejects_mismatched_packed_and_aux() { + let mut out = [0u8; 2]; + batch_packed_i4_16(&[0u64, 0], &[0i8], &mut out, |_, _| 0); + } + + #[test] + fn prefetch_hints_accept_any_pointer_without_dereferencing() { + prefetch_read_t0(core::ptr::null()); + prefetch_read_t1(usize::MAX as *const u8); + prefetch_read_t2(core::ptr::dangling()); + } +} diff --git a/src/simd_scalar.rs b/src/simd_scalar.rs index 6482b585..49ffb450 100644 --- a/src/simd_scalar.rs +++ b/src/simd_scalar.rs @@ -2087,33 +2087,26 @@ impl U64x8 { #[inline(always)] pub fn ternlog(self, b: Self, c: Self) -> Self { const { assert!(IMM >= 0 && IMM <= 255, "ternlog IMM is an 8-bit truth table") } - let (a, z) = (self, Self::splat(0)); - let mut r = z; - if IMM & 0x01 != 0 { - r = r | !a & !b & !c; + // GENERATED lowering (tools/gen_ternlog_bodies.py): Shannon-expand on `c` + // into two 2-input tables; <= 8 ops for any table in this vocabulary + // (and-not is `x & !y`, two ops), folded at compile time. + let t0: u8 = ((IMM & 1) | ((IMM >> 1) & 2) | ((IMM >> 2) & 4) | ((IMM >> 3) & 8)) as u8; + let t1: u8 = (((IMM >> 1) & 1) | ((IMM >> 2) & 2) | ((IMM >> 3) & 4) | ((IMM >> 4) & 8)) as u8; + if t0 == t1 { + ternlog_two_input_u64x8(t0, self, b) + } else if t0 == 0 { + c & ternlog_two_input_u64x8(t1, self, b) + } else if t1 == 0 { + ternlog_two_input_u64x8(t0, self, b) & !c + } else if t1 == (t0 ^ 0xF) { + c ^ ternlog_two_input_u64x8(t0, self, b) + } else if t0 == 0xF { + !c | ternlog_two_input_u64x8(t1, self, b) + } else if t1 == 0xF { + c | ternlog_two_input_u64x8(t0, self, b) + } else { + (ternlog_two_input_u64x8(t0, self, b) & !c) | (ternlog_two_input_u64x8(t1, self, b) & c) } - if IMM & 0x02 != 0 { - r = r | !a & !b & c; - } - if IMM & 0x04 != 0 { - r = r | !a & b & !c; - } - if IMM & 0x08 != 0 { - r = r | !a & b & c; - } - if IMM & 0x10 != 0 { - r = r | a & !b & !c; - } - if IMM & 0x20 != 0 { - r = r | a & !b & c; - } - if IMM & 0x40 != 0 { - r = r | a & b & !c; - } - if IMM & 0x80 != 0 { - r = r | a & b & c; - } - r } } @@ -2146,32 +2139,79 @@ impl U32x16 { #[inline(always)] pub fn ternlog(self, b: Self, c: Self) -> Self { const { assert!(IMM >= 0 && IMM <= 255, "ternlog IMM is an 8-bit truth table") } - let (a, z) = (self, Self::splat(0)); - let mut r = z; - if IMM & 0x01 != 0 { - r = r | !a & !b & !c; - } - if IMM & 0x02 != 0 { - r = r | !a & !b & c; - } - if IMM & 0x04 != 0 { - r = r | !a & b & !c; - } - if IMM & 0x08 != 0 { - r = r | !a & b & c; - } - if IMM & 0x10 != 0 { - r = r | a & !b & !c; + // GENERATED lowering (tools/gen_ternlog_bodies.py): Shannon-expand on `c` + // into two 2-input tables; <= 8 ops for any table in this vocabulary + // (and-not is `x & !y`, two ops), folded at compile time. + let t0: u8 = ((IMM & 1) | ((IMM >> 1) & 2) | ((IMM >> 2) & 4) | ((IMM >> 3) & 8)) as u8; + let t1: u8 = (((IMM >> 1) & 1) | ((IMM >> 2) & 2) | ((IMM >> 3) & 4) | ((IMM >> 4) & 8)) as u8; + if t0 == t1 { + ternlog_two_input_u32x16(t0, self, b) + } else if t0 == 0 { + c & ternlog_two_input_u32x16(t1, self, b) + } else if t1 == 0 { + ternlog_two_input_u32x16(t0, self, b) & !c + } else if t1 == (t0 ^ 0xF) { + c ^ ternlog_two_input_u32x16(t0, self, b) + } else if t0 == 0xF { + !c | ternlog_two_input_u32x16(t1, self, b) + } else if t1 == 0xF { + c | ternlog_two_input_u32x16(t0, self, b) + } else { + (ternlog_two_input_u32x16(t0, self, b) & !c) | (ternlog_two_input_u32x16(t1, self, b) & c) } - if IMM & 0x20 != 0 { - r = r | a & !b & c; - } - if IMM & 0x40 != 0 { - r = r | a & b & !c; - } - if IMM & 0x80 != 0 { - r = r | a & b & c; - } - r } } + +// GEN-TERNLOG-BEGIN (tools/gen_ternlog_bodies.py — regenerate, do not hand-edit) +/// GENERATED by `tools/gen_ternlog_bodies.py` — a 2-input Boolean function +/// by its 4-bit table (bit `k` = value at index `(a << 1) | b`), at most +/// two operations. `#[inline]` (not `always`): the 256-table test would +/// otherwise carry every arm's temporaries in one debug frame. +#[inline] +fn ternlog_two_input_u64x8(t: u8, a: U64x8, b: U64x8) -> U64x8 { + match t & 0xF { + 0x0 => U64x8::splat(0), + 0x1 => !(a | b), + 0x2 => !a & b, + 0x3 => !a, + 0x4 => a & !b, + 0x5 => !b, + 0x6 => a ^ b, + 0x7 => !(a & b), + 0x8 => a & b, + 0x9 => !(a ^ b), + 0xa => b, + 0xb => !a | b, + 0xc => a, + 0xd => a | !b, + 0xe => a | b, + _ => U64x8::splat(!0), + } +} + +/// GENERATED by `tools/gen_ternlog_bodies.py` — a 2-input Boolean function +/// by its 4-bit table (bit `k` = value at index `(a << 1) | b`), at most +/// two operations. `#[inline]` (not `always`): the 256-table test would +/// otherwise carry every arm's temporaries in one debug frame. +#[inline] +fn ternlog_two_input_u32x16(t: u8, a: U32x16, b: U32x16) -> U32x16 { + match t & 0xF { + 0x0 => U32x16::splat(0), + 0x1 => !(a | b), + 0x2 => !a & b, + 0x3 => !a, + 0x4 => a & !b, + 0x5 => !b, + 0x6 => a ^ b, + 0x7 => !(a & b), + 0x8 => a & b, + 0x9 => !(a ^ b), + 0xa => b, + 0xb => !a | b, + 0xc => a, + 0xd => a | !b, + 0xe => a | b, + _ => U32x16::splat(!0), + } +} +// GEN-TERNLOG-END diff --git a/src/simd_wasm.rs b/src/simd_wasm.rs index 5fb9f082..d43cad5a 100644 --- a/src/simd_wasm.rs +++ b/src/simd_wasm.rs @@ -75,7 +75,6 @@ pub mod wasm32_simd { // `U32x16` is the exception: it carries the ARX vocabulary (Add/BitXor/ // rotate_left) the ChaCha20 lane needs, so it is native here (`[U32x4; 4]`, // NEON-style) rather than the scalar fallback — see below. - pub use crate::simd::scalar::{I32x16, U64x8}; // ════════════════════════════════════════════════════════════════════ // F32x16 — 16 × f32 backed by 4 × v128 (f32x4 interpretation) @@ -295,7 +294,7 @@ pub mod wasm32_simd { for i in 0..16 { o[i] = a[i] as i32; } - I32x16(o) + I32x16::from_array(o) } } @@ -398,6 +397,27 @@ pub mod wasm32_simd { #[derive(Copy, Clone, Debug)] pub struct F32Mask16(pub u16); impl F32Mask16 { + /// The mask as a packed 16-bit bitmask, LSB-first (bit `i` = lane `i`). + /// The one representation-independent reading of a compare result: every + /// backend stores its mask differently (`__mmask16`, `u16`, + /// `core::simd::Mask`), so callers combine and inspect masks through this + /// rather than the tuple field (the `aabb` broadphase read `.0` directly + /// and did not compile on the portable backend — fixed 2026-09-14). + /// + /// # Examples + /// Bit `i` is lane `i`: with lanes 0 and 15 below the threshold the + /// `simd_lt` mask reads `0b1000_0000_0000_0001`. + /// ```rust,ignore + /// let mut a = [10.0f32; 16]; + /// a[0] = -1.0; + /// a[15] = -1.0; + /// let m = F32x16::from_array(a).simd_lt(F32x16::splat(0.0)); + /// assert_eq!(m.to_bitmask(), 0b1000_0000_0000_0001); + /// ``` + #[inline(always)] + pub fn to_bitmask(self) -> u16 { + self.0 + } #[inline(always)] pub fn select(self, true_val: F32x16, false_val: F32x16) -> F32x16 { let t = true_val.to_array(); @@ -604,13 +624,14 @@ pub mod wasm32_simd { for i in 0..8 { o[i] = a[i].to_bits(); } - U64x8(o) + U64x8::from_array(o) } #[inline(always)] pub fn from_bits(bits: U64x8) -> Self { + let b = bits.to_array(); let mut o = [0.0f64; 8]; for i in 0..8 { - o[i] = f64::from_bits(bits.0[i]); + o[i] = f64::from_bits(b[i]); } Self::from_array(o) } @@ -997,37 +1018,32 @@ pub mod wasm32_simd { #[inline(always)] pub fn ternlog(self, b: Self, c: Self) -> Self { const { assert!(IMM >= 0 && IMM <= 255, "ternlog IMM is an 8-bit truth table") } - let mut parts = [self.0[0].0; 4]; - for p in 0..4 { + // GENERATED lowering (tools/gen_ternlog_bodies.py), per 128-bit quad. + Self(core::array::from_fn(|p| { let (x, y, z) = (self.0[p].0, b.0[p].0, c.0[p].0); - let mut r = v128_xor(x, x); // zero - if IMM & 0x01 != 0 { - r = v128_or(r, v128_and(v128_not(x), v128_andnot(v128_not(y), z))); - } - if IMM & 0x02 != 0 { - r = v128_or(r, v128_and(v128_not(x), v128_andnot(z, y))); - } - if IMM & 0x04 != 0 { - r = v128_or(r, v128_and(v128_not(x), v128_andnot(y, z))); - } - if IMM & 0x08 != 0 { - r = v128_or(r, v128_and(v128_not(x), v128_and(y, z))); - } - if IMM & 0x10 != 0 { - r = v128_or(r, v128_and(x, v128_andnot(v128_not(y), z))); - } - if IMM & 0x20 != 0 { - r = v128_or(r, v128_and(x, v128_andnot(z, y))); - } - if IMM & 0x40 != 0 { - r = v128_or(r, v128_and(x, v128_andnot(y, z))); - } - if IMM & 0x80 != 0 { - r = v128_or(r, v128_and(x, v128_and(y, z))); - } - parts[p] = r; - } - Self([U32x4(parts[0]), U32x4(parts[1]), U32x4(parts[2]), U32x4(parts[3])]) + let t0: u8 = ((IMM & 1) | ((IMM >> 1) & 2) | ((IMM >> 2) & 4) | ((IMM >> 3) & 8)) as u8; + let t1: u8 = (((IMM >> 1) & 1) | ((IMM >> 2) & 2) | ((IMM >> 3) & 4) | ((IMM >> 4) & 8)) as u8; + U32x4({ + if t0 == t1 { + ternlog_two_input_v128(t0, x, y) + } else if t0 == 0 { + v128_and(z, ternlog_two_input_v128(t1, x, y)) + } else if t1 == 0 { + v128_andnot(ternlog_two_input_v128(t0, x, y), z) + } else if t1 == (t0 ^ 0xF) { + v128_xor(z, ternlog_two_input_v128(t0, x, y)) + } else if t0 == 0xF { + v128_or(v128_not(z), ternlog_two_input_v128(t1, x, y)) + } else if t1 == 0xF { + v128_or(z, ternlog_two_input_v128(t0, x, y)) + } else { + v128_or( + v128_andnot(ternlog_two_input_v128(t0, x, y), z), + v128_and(ternlog_two_input_v128(t1, x, y), z), + ) + } + }) + })) } /// `_mm256_unpacklo_epi32` per 256-bit half: within each 128-bit quad, @@ -1186,6 +1202,26 @@ pub mod wasm32_simd { o } + /// Wrapping horizontal sum (`u32x4_add` tree, four extracts) — the + /// same method the AVX2 / AVX-512 / scalar `U32x16` carry; missing on + /// this backend until the aarch64 codegen witness exposed the gap on + /// 2026-09-14 (the wasm twin was closed in the same pass). + /// + /// # Examples + /// ```rust,ignore + /// let v = U32x16::from_array(core::array::from_fn(|i| i as u32)); // 0..16 + /// assert_eq!(v.reduce_sum(), 120); + /// assert_eq!(U32x16::splat(u32::MAX).reduce_sum(), u32::MAX.wrapping_mul(16)); + /// ``` + #[inline(always)] + pub fn reduce_sum(self) -> u32 { + let t = u32x4_add(u32x4_add(self.0[0].0, self.0[1].0), u32x4_add(self.0[2].0, self.0[3].0)); + u32x4_extract_lane::<0>(t) + .wrapping_add(u32x4_extract_lane::<1>(t)) + .wrapping_add(u32x4_extract_lane::<2>(t)) + .wrapping_add(u32x4_extract_lane::<3>(t)) + } + /// Lane-wise left-rotate by `n` bits (ARX rotate), fanned over 4 lanes. #[inline(always)] pub fn rotate_left(self, n: u32) -> Self { @@ -1201,8 +1237,8 @@ pub mod wasm32_simd { /// /// Bit `i` of the result is set iff `self.lane(i) == other.lane(i)`. /// Bit order is **LSB-first**: lane `0` occupies bit `0`. Same - /// convention as `I32x16::cmpge_zero_mask` / `I32x16::gt_bitmask` - /// (which on wasm32 come from the scalar tier). + /// convention as the native wasm `I32x16::cmpge_zero_mask` / + /// `I32x16::gt_bitmask` below. /// /// Edge cases: equality is exact bitwise comparison over the full /// 32-bit range, so `u32::MAX` and `0` behave like any other value — @@ -1268,6 +1304,12 @@ pub mod wasm32_simd { pub type i8x16 = I8x16; #[allow(non_camel_case_types)] pub type u32x16 = U32x16; + /// Lowercase alias of the native wasm [`I32x16`] (travels with the type). + #[allow(non_camel_case_types)] + pub type i32x16 = I32x16; + /// Lowercase alias of the native wasm [`U64x8`] (travels with the type). + #[allow(non_camel_case_types)] + pub type u64x8 = U64x8; // ════════════════════════════════════════════════════════════════════ // Free hot-kernel functions — v128 counterparts to the NEON kernels in @@ -1702,4 +1744,585 @@ pub mod wasm32_simd { } } } + + // ════════════════════════════════════════════════════════════════════ + // Native U64x8 / I32x16 — the two lane types the mask family rides + // (2026-09-13, PR #306 five-flavour audit) + // ════════════════════════════════════════════════════════════════════ + // + // `simd_masking_ops` builds every bulk mask op on `U64x8` and the whole + // signed-compare family on `I32x16::gt_bitmask`; until this section both + // came from the SCALAR backend on wasm32, so the mask lane ran per-element + // loops while only the `U32x16` paths reached v128. Same `[quad; 4]` + // fan-out as `U32x16`; every op is the simd128 instruction per quad. The + // simd128 intrinsics are safe on the pinned toolchain — no `unsafe` here; + // quads are built with the `u64x2(..)` / `i32x4(..)` constructors and + // read back with `*_extract_lane`, so no raw-pointer loads either. + // Surface = the scalar backend's `impl_int_type!` set plus its `U64x8` / + // `I32x16` extras, signature for signature. + + /// 2×u64 in one `v128`. + #[derive(Copy, Clone)] + #[repr(transparent)] + pub struct U64x2(pub v128); + + /// 4×i32 in one `v128`. + #[derive(Copy, Clone)] + #[repr(transparent)] + pub struct I32x4(pub v128); + + /// 8×u64 backed by 4× `v128` (`[U64x2; 4]`). The packed-mask word lane. + #[derive(Copy, Clone)] + #[repr(align(64))] + pub struct U64x8(pub [U64x2; 4]); + + impl Default for U64x8 { + #[inline(always)] + fn default() -> Self { + Self::splat(0) + } + } + + impl U64x8 { + pub const LANES: usize = 8; + + /// Broadcast (`u64x2_splat` ×4). + #[inline(always)] + pub fn splat(v: u64) -> Self { + Self([U64x2(u64x2_splat(v)); 4]) + } + + /// All-zero lanes. + #[inline(always)] + pub fn zero() -> Self { + Self::splat(0) + } + + /// Load the first 8 elements of `s`. Panics if `s.len() < 8`. + #[inline(always)] + pub fn from_slice(s: &[u64]) -> Self { + assert!(s.len() >= 8); + Self([ + U64x2(u64x2(s[0], s[1])), + U64x2(u64x2(s[2], s[3])), + U64x2(u64x2(s[4], s[5])), + U64x2(u64x2(s[6], s[7])), + ]) + } + + #[inline(always)] + pub fn from_array(a: [u64; 8]) -> Self { + Self::from_slice(&a) + } + + #[inline(always)] + pub fn to_array(self) -> [u64; 8] { + let mut o = [0u64; 8]; + self.copy_to_slice(&mut o); + o + } + + /// Store the 8 lanes into the front of `s`. Panics if `s.len() < 8`. + #[inline(always)] + pub fn copy_to_slice(self, s: &mut [u64]) { + assert!(s.len() >= 8); + for p in 0..4 { + s[2 * p] = u64x2_extract_lane::<0>(self.0[p].0); + s[2 * p + 1] = u64x2_extract_lane::<1>(self.0[p].0); + } + } + + /// Wrapping horizontal sum: `i64x2_add` tree, two lane extracts. + #[inline(always)] + pub fn reduce_sum(self) -> u64 { + let t = i64x2_add(i64x2_add(self.0[0].0, self.0[1].0), i64x2_add(self.0[2].0, self.0[3].0)); + u64x2_extract_lane::<0>(t).wrapping_add(u64x2_extract_lane::<1>(t)) + } + + /// Lane-wise left-rotate by `n` bits, `n` mod 64 (`i64x2_shl` | `u64x2_shr`). + #[inline(always)] + pub fn rotate_left(self, n: u32) -> Self { + let n = n % 64; + if n == 0 { + return self; + } + Self(core::array::from_fn(|p| U64x2(v128_or(i64x2_shl(self.0[p].0, n), u64x2_shr(self.0[p].0, 64 - n))))) + } + + /// Lane-wise right-rotate — `rotr(n) == rotl(64 - n)` exactly. + #[inline(always)] + pub fn rotate_right(self, n: u32) -> Self { + let n = n % 64; + if n == 0 { + return self; + } + self.rotate_left(64 - n) + } + + /// Lane-wise population count: `i8x16_popcnt`, then the pairwise + /// widening adds up to 32-bit halves, then the two halves of each u64 + /// summed (`u64x2_shr` 32 + masked add). + #[inline(always)] + pub fn popcnt(self) -> Self { + Self(core::array::from_fn(|p| { + let bytes = i8x16_popcnt(self.0[p].0); + let halves = u32x4_extadd_pairwise_u16x8(u16x8_extadd_pairwise_u8x16(bytes)); + let lo = v128_and(halves, u64x2_splat(0xFFFF_FFFF)); + U64x2(i64x2_add(lo, u64x2_shr(halves, 32))) + })) + } + + /// XOR lane-wise, popcount, sum all 8 lanes — the 512-bit Hamming distance. + #[inline(always)] + pub fn xor_popcount(self, other: Self) -> u64 { + (self ^ other).popcnt().reduce_sum() + } + + /// Set difference: `self & !other` — `v128_andnot(a, b)` IS `a & !b` + /// (unlike Intel's `andnot`). Same direction as every backend. + #[inline(always)] + pub fn andnot(self, other: Self) -> Self { + Self(core::array::from_fn(|p| U64x2(v128_andnot(self.0[p].0, other.0[p].0)))) + } + + /// Any 3-input boolean function of `self`, `b` and `c`, selected by the + /// const truth-table immediate `IMM` — Intel's VPTERNLOG convention + /// (`index = (self << 2) | (b << 1) | c`, result bit = `(IMM >> index) & 1`), + /// matched exactly by every backend. Only `0..=255` is legal + /// (compile-time assert). Named immediates: `crate::simd::ternlog`. + /// The body is generated (`tools/gen_ternlog_bodies.py`): a + /// Shannon-expanded ladder in `v128_and/or/xor/andnot` per quad. + #[inline(always)] + pub fn ternlog(self, b: Self, c: Self) -> Self { + const { assert!(IMM >= 0 && IMM <= 255, "ternlog IMM is an 8-bit truth table") } + // GENERATED lowering (tools/gen_ternlog_bodies.py), per 128-bit quad. + Self(core::array::from_fn(|p| { + let (x, y, z) = (self.0[p].0, b.0[p].0, c.0[p].0); + let t0: u8 = ((IMM & 1) | ((IMM >> 1) & 2) | ((IMM >> 2) & 4) | ((IMM >> 3) & 8)) as u8; + let t1: u8 = (((IMM >> 1) & 1) | ((IMM >> 2) & 2) | ((IMM >> 3) & 4) | ((IMM >> 4) & 8)) as u8; + U64x2({ + if t0 == t1 { + ternlog_two_input_v128(t0, x, y) + } else if t0 == 0 { + v128_and(z, ternlog_two_input_v128(t1, x, y)) + } else if t1 == 0 { + v128_andnot(ternlog_two_input_v128(t0, x, y), z) + } else if t1 == (t0 ^ 0xF) { + v128_xor(z, ternlog_two_input_v128(t0, x, y)) + } else if t0 == 0xF { + v128_or(v128_not(z), ternlog_two_input_v128(t1, x, y)) + } else if t1 == 0xF { + v128_or(z, ternlog_two_input_v128(t0, x, y)) + } else { + v128_or( + v128_andnot(ternlog_two_input_v128(t0, x, y), z), + v128_and(ternlog_two_input_v128(t1, x, y), z), + ) + } + }) + })) + } + } + + impl Add for U64x8 { + type Output = Self; + #[inline(always)] + fn add(self, r: Self) -> Self { + Self(core::array::from_fn(|p| U64x2(i64x2_add(self.0[p].0, r.0[p].0)))) + } + } + impl Sub for U64x8 { + type Output = Self; + #[inline(always)] + fn sub(self, r: Self) -> Self { + Self(core::array::from_fn(|p| U64x2(i64x2_sub(self.0[p].0, r.0[p].0)))) + } + } + impl AddAssign for U64x8 { + #[inline(always)] + fn add_assign(&mut self, r: Self) { + *self = *self + r; + } + } + impl SubAssign for U64x8 { + #[inline(always)] + fn sub_assign(&mut self, r: Self) { + *self = *self - r; + } + } + impl core::ops::BitAnd for U64x8 { + type Output = Self; + #[inline(always)] + fn bitand(self, r: Self) -> Self { + Self(core::array::from_fn(|p| U64x2(v128_and(self.0[p].0, r.0[p].0)))) + } + } + impl core::ops::BitOr for U64x8 { + type Output = Self; + #[inline(always)] + fn bitor(self, r: Self) -> Self { + Self(core::array::from_fn(|p| U64x2(v128_or(self.0[p].0, r.0[p].0)))) + } + } + impl BitXor for U64x8 { + type Output = Self; + #[inline(always)] + fn bitxor(self, r: Self) -> Self { + Self(core::array::from_fn(|p| U64x2(v128_xor(self.0[p].0, r.0[p].0)))) + } + } + impl core::ops::BitAndAssign for U64x8 { + #[inline(always)] + fn bitand_assign(&mut self, r: Self) { + *self = *self & r; + } + } + impl core::ops::BitOrAssign for U64x8 { + #[inline(always)] + fn bitor_assign(&mut self, r: Self) { + *self = *self | r; + } + } + impl core::ops::BitXorAssign for U64x8 { + #[inline(always)] + fn bitxor_assign(&mut self, r: Self) { + *self = *self ^ r; + } + } + impl core::ops::Not for U64x8 { + type Output = Self; + #[inline(always)] + fn not(self) -> Self { + Self(core::array::from_fn(|p| U64x2(v128_not(self.0[p].0)))) + } + } + /// Lane-wise `self << rhs` with PER-LANE counts. simd128 has only a uniform + /// shift, so this one operator goes through lane extracts; the scalar + /// backend's `<<` panics on a count of 64 or more in debug builds, so + /// callers already stay inside `0..64`. + impl core::ops::Shl for U64x8 { + type Output = Self; + #[inline(always)] + fn shl(self, r: Self) -> Self { + let (a, n) = (self.to_array(), r.to_array()); + Self::from_array(core::array::from_fn(|i| a[i].wrapping_shl(n[i] as u32))) + } + } + /// Lane-wise `self >> rhs` with per-lane counts (see `Shl`). + impl core::ops::Shr for U64x8 { + type Output = Self; + #[inline(always)] + fn shr(self, r: Self) -> Self { + let (a, n) = (self.to_array(), r.to_array()); + Self::from_array(core::array::from_fn(|i| a[i].wrapping_shr(n[i] as u32))) + } + } + impl PartialEq for U64x8 { + #[inline(always)] + fn eq(&self, other: &Self) -> bool { + (0..4).all(|p| i64x2_all_true(i64x2_eq(self.0[p].0, other.0[p].0))) + } + } + impl fmt::Debug for U64x8 { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "U64x8({:?})", self.to_array()) + } + } + + /// 16×i32 backed by 4× `v128` (`[I32x4; 4]`). The signed-compare lane: + /// `gt_bitmask` / `cmpge_zero_mask` are `i32x4_gt` / `i32x4_ge` + + /// `i32x4_bitmask` (no per-lane loop). + #[derive(Copy, Clone)] + #[repr(align(64))] + pub struct I32x16(pub [I32x4; 4]); + + impl Default for I32x16 { + #[inline(always)] + fn default() -> Self { + Self::splat(0) + } + } + + impl I32x16 { + pub const LANES: usize = 16; + + /// Broadcast (`i32x4_splat` ×4). + #[inline(always)] + pub fn splat(v: i32) -> Self { + Self([I32x4(i32x4_splat(v)); 4]) + } + + /// All-zero lanes. + #[inline(always)] + pub fn zero() -> Self { + Self::splat(0) + } + + /// Load the first 16 elements of `s`. Panics if `s.len() < 16`. + #[inline(always)] + pub fn from_slice(s: &[i32]) -> Self { + assert!(s.len() >= 16); + Self(core::array::from_fn(|p| I32x4(i32x4(s[4 * p], s[4 * p + 1], s[4 * p + 2], s[4 * p + 3])))) + } + + #[inline(always)] + pub fn from_array(a: [i32; 16]) -> Self { + Self::from_slice(&a) + } + + #[inline(always)] + pub fn to_array(self) -> [i32; 16] { + let mut o = [0i32; 16]; + self.copy_to_slice(&mut o); + o + } + + /// Store the 16 lanes into the front of `s`. Panics if `s.len() < 16`. + #[inline(always)] + pub fn copy_to_slice(self, s: &mut [i32]) { + assert!(s.len() >= 16); + for p in 0..4 { + let q = self.0[p].0; + s[4 * p] = i32x4_extract_lane::<0>(q); + s[4 * p + 1] = i32x4_extract_lane::<1>(q); + s[4 * p + 2] = i32x4_extract_lane::<2>(q); + s[4 * p + 3] = i32x4_extract_lane::<3>(q); + } + } + + /// Wrapping horizontal sum (`i32x4_add` tree, four extracts). + #[inline(always)] + pub fn reduce_sum(self) -> i32 { + let t = i32x4_add(i32x4_add(self.0[0].0, self.0[1].0), i32x4_add(self.0[2].0, self.0[3].0)); + i32x4_extract_lane::<0>(t) + .wrapping_add(i32x4_extract_lane::<1>(t)) + .wrapping_add(i32x4_extract_lane::<2>(t)) + .wrapping_add(i32x4_extract_lane::<3>(t)) + } + + /// Minimum over all 16 lanes (`i32x4_min` tree, four extracts). + #[inline(always)] + pub fn reduce_min(self) -> i32 { + let t = i32x4_min(i32x4_min(self.0[0].0, self.0[1].0), i32x4_min(self.0[2].0, self.0[3].0)); + i32x4_extract_lane::<0>(t) + .min(i32x4_extract_lane::<1>(t)) + .min(i32x4_extract_lane::<2>(t)) + .min(i32x4_extract_lane::<3>(t)) + } + + /// Maximum over all 16 lanes (`i32x4_max` tree, four extracts). + #[inline(always)] + pub fn reduce_max(self) -> i32 { + let t = i32x4_max(i32x4_max(self.0[0].0, self.0[1].0), i32x4_max(self.0[2].0, self.0[3].0)); + i32x4_extract_lane::<0>(t) + .max(i32x4_extract_lane::<1>(t)) + .max(i32x4_extract_lane::<2>(t)) + .max(i32x4_extract_lane::<3>(t)) + } + + /// Lane-wise minimum (`i32x4_min`). + #[inline(always)] + pub fn simd_min(self, other: Self) -> Self { + Self(core::array::from_fn(|p| I32x4(i32x4_min(self.0[p].0, other.0[p].0)))) + } + + /// Lane-wise maximum (`i32x4_max`). + #[inline(always)] + pub fn simd_max(self, other: Self) -> Self { + Self(core::array::from_fn(|p| I32x4(i32x4_max(self.0[p].0, other.0[p].0)))) + } + + /// Lane-wise `i32 → f32` (`f32x4_convert_i32x4` per quad). + #[inline(always)] + pub fn cast_f32(self) -> F32x16 { + F32x16(core::array::from_fn(|p| f32x4_convert_i32x4(self.0[p].0))) + } + + /// Lane-wise absolute value (`i32x4_abs`; `i32::MIN` wraps to itself, + /// the release-mode behaviour of the scalar backend). + #[inline(always)] + pub fn abs(self) -> Self { + Self(core::array::from_fn(|p| I32x4(i32x4_abs(self.0[p].0)))) + } + + /// Sign-extend the first 16 `i16` of `s` (`i32x4_extend_low/high_i16x8` + /// over two `i16x8` quads). Panics if `s.len() < 16`. + #[inline(always)] + pub fn from_i16_slice(s: &[i16]) -> Self { + assert!(s.len() >= 16); + let lo = i16x8(s[0], s[1], s[2], s[3], s[4], s[5], s[6], s[7]); + let hi = i16x8(s[8], s[9], s[10], s[11], s[12], s[13], s[14], s[15]); + Self([ + I32x4(i32x4_extend_low_i16x8(lo)), + I32x4(i32x4_extend_high_i16x8(lo)), + I32x4(i32x4_extend_low_i16x8(hi)), + I32x4(i32x4_extend_high_i16x8(hi)), + ]) + } + + /// Truncate each lane to `i16` (low 16 bits, like `as i16` — NOT the + /// saturating `i16x8_narrow_i32x4`, which would change values). + #[inline(always)] + pub fn to_i16_array(self) -> [i16; 16] { + let a = self.to_array(); + core::array::from_fn(|i| a[i] as i16) + } + + /// Bit `i` set iff lane `i >= 0` (`i32x4_ge` vs zero + `i32x4_bitmask`), LSB-first. + #[inline(always)] + pub fn cmpge_zero_mask(self) -> u16 { + let z = i32x4_splat(0); + let mut m = 0u16; + for p in 0..4 { + m |= (i32x4_bitmask(i32x4_ge(self.0[p].0, z)) as u16) << (4 * p); + } + m + } + + /// Lane-wise **signed** greater-than as a packed 16-bit bitmask + /// (`i32x4_gt` + `i32x4_bitmask`). Bit `i` set iff + /// `self.lane(i) > other.lane(i)`, LSB-first; exact at `i32::MIN` / + /// `i32::MAX`; signed, never bit-pattern. Agrees bit-for-bit with the + /// scalar correctness anchor. + #[inline(always)] + pub fn gt_bitmask(self, other: Self) -> u16 { + let mut m = 0u16; + for p in 0..4 { + m |= (i32x4_bitmask(i32x4_gt(self.0[p].0, other.0[p].0)) as u16) << (4 * p); + } + m + } + } + + impl Add for I32x16 { + type Output = Self; + #[inline(always)] + fn add(self, r: Self) -> Self { + Self(core::array::from_fn(|p| I32x4(i32x4_add(self.0[p].0, r.0[p].0)))) + } + } + impl Sub for I32x16 { + type Output = Self; + #[inline(always)] + fn sub(self, r: Self) -> Self { + Self(core::array::from_fn(|p| I32x4(i32x4_sub(self.0[p].0, r.0[p].0)))) + } + } + impl AddAssign for I32x16 { + #[inline(always)] + fn add_assign(&mut self, r: Self) { + *self = *self + r; + } + } + impl SubAssign for I32x16 { + #[inline(always)] + fn sub_assign(&mut self, r: Self) { + *self = *self - r; + } + } + impl Mul for I32x16 { + type Output = Self; + #[inline(always)] + fn mul(self, r: Self) -> Self { + Self(core::array::from_fn(|p| I32x4(i32x4_mul(self.0[p].0, r.0[p].0)))) + } + } + impl MulAssign for I32x16 { + #[inline(always)] + fn mul_assign(&mut self, r: Self) { + *self = *self * r; + } + } + impl Neg for I32x16 { + type Output = Self; + #[inline(always)] + fn neg(self) -> Self { + Self(core::array::from_fn(|p| I32x4(i32x4_neg(self.0[p].0)))) + } + } + impl core::ops::BitAnd for I32x16 { + type Output = Self; + #[inline(always)] + fn bitand(self, r: Self) -> Self { + Self(core::array::from_fn(|p| I32x4(v128_and(self.0[p].0, r.0[p].0)))) + } + } + impl core::ops::BitOr for I32x16 { + type Output = Self; + #[inline(always)] + fn bitor(self, r: Self) -> Self { + Self(core::array::from_fn(|p| I32x4(v128_or(self.0[p].0, r.0[p].0)))) + } + } + impl BitXor for I32x16 { + type Output = Self; + #[inline(always)] + fn bitxor(self, r: Self) -> Self { + Self(core::array::from_fn(|p| I32x4(v128_xor(self.0[p].0, r.0[p].0)))) + } + } + impl core::ops::BitAndAssign for I32x16 { + #[inline(always)] + fn bitand_assign(&mut self, r: Self) { + *self = *self & r; + } + } + impl core::ops::BitOrAssign for I32x16 { + #[inline(always)] + fn bitor_assign(&mut self, r: Self) { + *self = *self | r; + } + } + impl core::ops::BitXorAssign for I32x16 { + #[inline(always)] + fn bitxor_assign(&mut self, r: Self) { + *self = *self ^ r; + } + } + impl core::ops::Not for I32x16 { + type Output = Self; + #[inline(always)] + fn not(self) -> Self { + Self(core::array::from_fn(|p| I32x4(v128_not(self.0[p].0)))) + } + } + impl PartialEq for I32x16 { + #[inline(always)] + fn eq(&self, other: &Self) -> bool { + (0..4).all(|p| i32x4_all_true(i32x4_eq(self.0[p].0, other.0[p].0))) + } + } + impl fmt::Debug for I32x16 { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "I32x16({:?})", self.to_array()) + } + } + + // GEN-TERNLOG-BEGIN (tools/gen_ternlog_bodies.py — regenerate, do not hand-edit) + /// GENERATED by `tools/gen_ternlog_bodies.py` — a 2-input Boolean function + /// by its 4-bit table (bit `k` = value at index `(a << 1) | b`), at most + /// two operations. `#[inline]` (not `always`): the 256-table test would + /// otherwise carry every arm's temporaries in one debug frame. + #[inline] + fn ternlog_two_input_v128(t: u8, a: v128, b: v128) -> v128 { + match t & 0xF { + 0x0 => u32x4_splat(0), + 0x1 => v128_not(v128_or(a, b)), + 0x2 => v128_and(v128_not(a), b), + 0x3 => v128_not(a), + 0x4 => v128_andnot(a, b), + 0x5 => v128_not(b), + 0x6 => v128_xor(a, b), + 0x7 => v128_not(v128_and(a, b)), + 0x8 => v128_and(a, b), + 0x9 => v128_not(v128_xor(a, b)), + 0xa => b, + 0xb => v128_or(v128_not(a), b), + 0xc => a, + 0xd => v128_or(a, v128_not(b)), + 0xe => v128_or(a, b), + _ => u32x4_splat(!0), + } + } + // GEN-TERNLOG-END } diff --git a/tools/gen_ternlog_bodies.py b/tools/gen_ternlog_bodies.py new file mode 100644 index 00000000..f32a99cb --- /dev/null +++ b/tools/gen_ternlog_bodies.py @@ -0,0 +1,326 @@ +#!/usr/bin/env python3 +"""Generate the backend-local `ternlog::` lowering bodies. + +THE BACKEND LAW (operator, 2026-09-13): ndarray exposes ONE architecture- +agnostic semantic API, selected at compile time into COMPLETE PEER +implementations in simd_avx512.rs / simd_avx2.rs / simd_neon.rs / +simd_wasm.rs / simd_scalar.rs. No shared runtime polyfill sits under them. +Shared *tests* and shared *generated truth-table logic* are fine; a common +implementation function the backends delegate into is not. + +This script is that shared logic. It derives, once, the Shannon lowering of +an 8-bit truth table into a minimal Boolean DAG, checks it against a +bit-serial reference for all 256 tables, and then PRINTS each backend's body +in that backend's own vocabulary (operator traits on the array-backed lane +types; `vandq/vorrq/veorq/vbicq_u{32,64}` NEON intrinsics per 128-bit quad — +NOT a per-lane `u32` loop, which LLVM scalarised (536 scalar / 4 vector ops +measured); `v128_*` intrinsics for WASM). +The emitted text is pasted into the backend file between GEN markers by +`--apply`; it is committed source, and the generator is its provenance. + +Lowering: with index `(a << 2) | (b << 1) | c`, the even bits of IMM are the +2-input table T0(a,b) (c = 0) and the odd bits T1(a,b) (c = 1); + f = (!c & T0) | (c & T1) +with every 2-input table a <= 2-op closed form and six collapse shapes of the +outer combination. Worst case 7 ops where the vocabulary has a native and-not +(NEON `vbic`, WASM `v128.andnot`), 8 where and-not is spelled `x & !y` (the +avx2/scalar operator vocabularies); the naive 8-minterm form was up to 36. +The count is ASSERTED below (`max_ops`), not just stated. +""" +import re, sys, pathlib + +ROOT = pathlib.Path(__file__).resolve().parent.parent / "src" + +# 2-input table (bit k = value at index (a<<1)|b) -> expression AST +TWO = { + 0x0: ("zero",), 0x1: ("not", ("or", "a", "b")), 0x2: ("and", ("not", "a"), "b"), + 0x3: ("not", "a"), 0x4: ("andnot", "a", "b"), 0x5: ("not", "b"), 0x6: ("xor", "a", "b"), + 0x7: ("not", ("and", "a", "b")), 0x8: ("and", "a", "b"), 0x9: ("not", ("xor", "a", "b")), + 0xA: "b", 0xB: ("or", ("not", "a"), "b"), 0xC: "a", 0xD: ("or", "a", ("not", "b")), + 0xE: ("or", "a", "b"), 0xF: ("ones",), +} + +def ev(ast, a, b, c=None): + if isinstance(ast, str): + return {"a": a, "b": b, "c": c, "g0": None, "g1": None}[ast] + k = ast[0] + if k == "zero": return 0 + if k == "ones": return 0xFFFFFFFFFFFFFFFF + if k == "not": return ev(ast[1], a, b, c) ^ 0xFFFFFFFFFFFFFFFF + x, y = ev(ast[1], a, b, c), ev(ast[2], a, b, c) + return {"and": x & y, "or": x | y, "xor": x ^ y, "andnot": x & (y ^ 0xFFFFFFFFFFFFFFFF)}[k] + +def reference(imm, a, b, c): + r = 0 + for bit in range(64): + idx = (((a >> bit) & 1) << 2) | (((b >> bit) & 1) << 1) | ((c >> bit) & 1) + r |= ((imm >> idx) & 1) << bit + return r + +def halves(imm): + t0 = sum(((imm >> (2 * k)) & 1) << k for k in range(4)) + t1 = sum(((imm >> (2 * k + 1)) & 1) << k for k in range(4)) + return t0, t1 + +# self-check: every table, adversarial operands covering all 8 index combos +A, B, C = 0xF0F0F0F0F0F0F0F0, 0xCCCCCCCCCCCCCCCC, 0xAAAAAAAAAAAAAAAA +def lowered_value(imm): + t0, t1 = halves(imm) + g0, g1 = ev(TWO[t0], A, B), ev(TWO[t1], A, B) + NOT = lambda x: x ^ 0xFFFFFFFFFFFFFFFF + if t0 == t1: return g0 + if t0 == 0: return C & g1 + if t1 == 0: return g0 & NOT(C) + if t1 == (t0 ^ 0xF): return C ^ g0 + if t0 == 0xF: return NOT(C) | g1 + if t1 == 0xF: return C | g0 + return (g0 & NOT(C)) | (g1 & C) +for imm in range(256): + assert lowered_value(imm) == reference(imm, A, B, C), f"lowering wrong at {imm:#04x}" + +# per-backend printers: (name, vocabulary) — vocabulary maps AST node -> source +def op_printer(kind_and, kind_or, kind_xor, kind_not, kind_andnot, zero, ones): + def p(ast): + if isinstance(ast, str): return ast + k = ast[0] + if k == "zero": return zero + if k == "ones": return ones + if k == "not": return kind_not(p(ast[1])) + x, y = p(ast[1]), p(ast[2]) + return {"and": kind_and, "or": kind_or, "xor": kind_xor, "andnot": kind_andnot}[k](x, y) + return p + +OPS = op_printer(lambda x,y: f"({x} & {y})", lambda x,y: f"({x} | {y})", lambda x,y: f"({x} ^ {y})", + lambda x: f"!{x}", lambda x,y: f"({x} & !{y})", "ZERO", "ONES") +NEON = op_printer(lambda x,y: f"vandq_u32({x}, {y})", lambda x,y: f"vorrq_u32({x}, {y})", + lambda x,y: f"veorq_u32({x}, {y})", lambda x: f"vmvnq_u32({x})", + lambda x,y: f"vbicq_u32({x}, {y})", "vdupq_n_u32(0)", "vdupq_n_u32(!0)") +NEON64 = op_printer(lambda x,y: f"vandq_u64({x}, {y})", lambda x,y: f"vorrq_u64({x}, {y})", + lambda x,y: f"veorq_u64({x}, {y})", lambda x: f"veorq_u64({x}, vdupq_n_u64(!0))", + lambda x,y: f"vbicq_u64({x}, {y})", "vdupq_n_u64(0)", "vdupq_n_u64(!0)") +WASM = op_printer(lambda x,y: f"v128_and({x}, {y})", lambda x,y: f"v128_or({x}, {y})", + lambda x,y: f"v128_xor({x}, {y})", lambda x: f"v128_not({x})", + lambda x,y: f"v128_andnot({x}, {y})", "u32x4_splat(0)", "u32x4_splat(!0)") + +def top(src): + """Strip one redundant outer paren layer (rustc `unused_parens` is a warning).""" + if src.startswith("(") and src.endswith(")"): + depth = 0 + for i, ch in enumerate(src): + depth += (ch == "(") - (ch == ")") + if depth == 0 and i < len(src) - 1: + return src + return src[1:-1] + return src + +def two_input_fn(name, ty, printer, zero, ones, indent="", attrs=(), unsafe_reason=None): + lines = [f"{indent}/// GENERATED by `tools/gen_ternlog_bodies.py` — a 2-input Boolean function", + f"{indent}/// by its 4-bit table (bit `k` = value at index `(a << 1) | b`), at most", + f"{indent}/// two operations. `#[inline]` (not `always`): the 256-table test would", + f"{indent}/// otherwise carry every arm's temporaries in one debug frame."] + lines += [f"{indent}{a}" for a in attrs] + lines += [f"{indent}#[inline]", + f"{indent}fn {name}(t: u8, a: {ty}, b: {ty}) -> {ty} {{"] + inner = indent + (" " if unsafe_reason else "") + if unsafe_reason: + for k, part in enumerate(unsafe_reason.split("\n")): + lines.append(f"{indent} // {'SAFETY: ' if k == 0 else ''}{part}") + lines.append(f"{indent} unsafe {{") + lines.append(f"{inner} match t & 0xF {{") + for k in range(16): + src = top(printer(TWO[k]).replace("ZERO", zero).replace("ONES", ones)) + lines.append(f"{inner} {'_' if k == 15 else f'{k:#03x}'} => {src},") + lines.append(f"{inner} }}") + if unsafe_reason: + lines.append(f"{indent} }}") + lines.append(f"{indent}}}") + return "\n".join(lines) + +def ladder(g, a, b, c, and_, or_, xor_, not_, andnot, indent, bind_indent=None): + """The outer Shannon combination, branching only on IMM-derived values (let-bound: a `const` item + cannot name the enclosing fn's IMM; after monomorphization these fold identically). + Returns (bindings, body) so a backend can place the two plain-integer `let`s OUTSIDE + its intrinsic `unsafe` block — the block then wraps intrinsic calls and nothing else.""" + bi = indent if bind_indent is None else bind_indent + B = [f"{bi}let t0: u8 = ((IMM & 1) | ((IMM >> 1) & 2) | ((IMM >> 2) & 4) | ((IMM >> 3) & 8)) as u8;", + f"{bi}let t1: u8 = (((IMM >> 1) & 1) | ((IMM >> 2) & 2) | ((IMM >> 3) & 4) | ((IMM >> 4) & 8)) as u8;"] + L = [] + L.append(f"{indent}if t0 == t1 {{") + L.append(f"{indent} {g}(t0, {a}, {b})") + L.append(f"{indent}}} else if t0 == 0 {{") + L.append(f"{indent} {top(and_(c, f'{g}(t1, {a}, {b})'))}") + L.append(f"{indent}}} else if t1 == 0 {{") + L.append(f"{indent} {top(andnot(f'{g}(t0, {a}, {b})', c))}") + L.append(f"{indent}}} else if t1 == (t0 ^ 0xF) {{") + L.append(f"{indent} {top(xor_(c, f'{g}(t0, {a}, {b})'))}") + L.append(f"{indent}}} else if t0 == 0xF {{") + L.append(f"{indent} {top(or_(not_(c), f'{g}(t1, {a}, {b})'))}") + L.append(f"{indent}}} else if t1 == 0xF {{") + L.append(f"{indent} {top(or_(c, f'{g}(t0, {a}, {b})'))}") + L.append(f"{indent}}} else {{") + L.append(f"{indent} {top(or_(andnot(f'{g}(t0, {a}, {b})', c), and_(f'{g}(t1, {a}, {b})', c)))}") + L.append(f"{indent}}}") + return "\n".join(B), "\n".join(L) + +def count_ops(src): + """Operator/intrinsic count of one emitted expression (the metric the docs quote).""" + return len(re.findall(r"[&|^]|!(?=[a-z(])|\bv(?:and|orr|eor|bic)q_u(?:32|64)\b|\bv128_(?:and|or|xor|not|andnot)\b|vdupq_n_u64\(!0\)", src)) + +OPL = dict(and_=lambda x,y: f"({x} & {y})", or_=lambda x,y: f"({x} | {y})", xor_=lambda x,y: f"({x} ^ {y})", + not_=lambda x: f"!{x}", andnot=lambda x,y: f"({x} & !{y})") +NEONL = dict(and_=lambda x,y: f"vandq_u32({x}, {y})", or_=lambda x,y: f"vorrq_u32({x}, {y})", + xor_=lambda x,y: f"veorq_u32({x}, {y})", not_=lambda x: f"vmvnq_u32({x})", + andnot=lambda x,y: f"vbicq_u32({x}, {y})") +NEON64L = dict(and_=lambda x,y: f"vandq_u64({x}, {y})", or_=lambda x,y: f"vorrq_u64({x}, {y})", + xor_=lambda x,y: f"veorq_u64({x}, {y})", not_=lambda x: f"veorq_u64({x}, vdupq_n_u64(!0))", + andnot=lambda x,y: f"vbicq_u64({x}, {y})") +WASML = dict(and_=lambda x,y: f"v128_and({x}, {y})", or_=lambda x,y: f"v128_or({x}, {y})", + xor_=lambda x,y: f"v128_xor({x}, {y})", not_=lambda x: f"v128_not({x})", + andnot=lambda x,y: f"v128_andnot({x}, {y})") + +BEGIN = "// GEN-TERNLOG-BEGIN (tools/gen_ternlog_bodies.py — regenerate, do not hand-edit)" +END = "// GEN-TERNLOG-END" + +def body_lane_type(ty): + """avx2 / scalar: the array-backed lane types own & | ^ ! and splat.""" + helper = f"ternlog_two_input_{ty.lower()}" + return ladder(helper, "self", "b", "c", indent=" ", **OPL), \ + two_input_fn(helper, ty, OPS, f"{ty}::splat(0)", f"{ty}::splat(!0)") + +MARK = re.compile(r"^( *)// GENERATED lowering \((?:tools/gen_ternlog_bodies\.py|regenerating)\)[^\n]*\n", re.M) + +# Where each generated body ENDS (the last line the generator itself emits), by +# shape. A freshly-written `(regenerating)` stub always ends at its +# `Self::from_array(o)` line, whatever the backend. +ARRAY_BODY_END = r"\n \}\n" # avx2/scalar: the ladder's final `}` (the fn's own `}` is kept) +NEON_BODY_END = r"\n \}\)\)\n" # `Self(core::array::from_fn(|p| { ... }))` +WASM_BODY_END = r"\n \}\)\)\n" # same shape, one module level deeper +STUB_END = r"\n *Self::from_array\(o\)\n" + +def apply(path, replacements, appendix, inside_module=None, write=True): + s = path.read_text() + for (sig_re, end_re), new in replacements: + m = sig_re.search(s) + assert m, (path.name, sig_re.pattern[-80:]) + ca = s.index("\n", s.index("const { assert!(IMM", m.end())) + 1 + mk = MARK.search(s, ca) + assert mk and mk.start() < ca + 400, (path.name, "no GENERATED marker after the const assert") + is_stub = "(regenerating)" in mk.group(0) + endm = re.compile(STUB_END if is_stub else end_re).search(s, mk.end()) + assert endm, (path.name, "body end not found") + # Replace from the marker through the END of the matched closer; `new` + # carries its own closer. Text before the marker (nothing but the + # const assert) and after the closer (the fn's own `}` where the body + # did not include it) is kept verbatim. + s = s[:mk.start()] + new + s[endm.end():] + s = re.sub(r"\nimpl crate::simd_ternlog_lower::TernlogLanes for \w+ \{.*?\n\}\n", "\n", s, flags=re.S) + if BEGIN in s: + i, j = s.index(BEGIN), s.index(END) + len(END) + s = s[:i].rstrip("\n") + "\n" + s[j:].lstrip("\n") + s = s.rstrip("\n") + "\n" + block = BEGIN + "\n" + appendix + "\n" + END + if inside_module is None: + s = s.rstrip("\n") + "\n\n" + block + "\n" + else: + head = s.index(inside_module) + close = s.index("\n}\n", head) + s = s[:close].rstrip("\n") + "\n\n" + block + "\n" + s[close:] + if write: + path.write_text(s) + return s + +def fn_sig(ty_impl_re, fn_indent): + """Regex for the `ternlog` signature INSIDE the given impl: from `impl {` to the first + ternlog signature, with no other `impl ` line in between (so an earlier `impl ` block + without a ternlog cannot capture a later type's fn).""" + return re.compile(ty_impl_re + r"(?:(?!\nimpl |\n impl ).)*?" + re.escape(fn_indent + "pub fn ternlog(self, b: Self, c: Self) -> Self {"), re.S) + +def main(write, check=False): + out = {} + worst = {} + # ── avx2 + scalar: U64x8 and U32x16 (array lanes, operator vocabulary) ── + for fname in ("simd_avx2.rs", "simd_scalar.rs"): + reps, helpers = [], [] + for ty in ("U64x8", "U32x16"): + helper = f"ternlog_two_input_{ty.lower()}" + binds, lad = ladder(helper, "self", "b", "c", indent=" ", **OPL) + body = (" // GENERATED lowering (tools/gen_ternlog_bodies.py): Shannon-expand on `c`\n" + " // into two 2-input tables; <= 8 ops for any table in this vocabulary\n" + " // (and-not is `x & !y`, two ops), folded at compile time.\n" + binds + "\n" + lad + "\n") + reps.append(((fn_sig(rf"impl {ty} \{{", " "), ARRAY_BODY_END), body)) + helpers.append(two_input_fn(helper, ty, OPS, f"{ty}::splat(0)", f"{ty}::splat(!0)")) + worst[(fname, ty)] = 8 + out[fname] = (reps, "\n\n".join(helpers), None) + # ── neon: U32x16 per-quad uint32x4_t and U64x8 per-quad uint64x2_t ── + NEON_SAFETY = ("NEON is a baseline feature of every aarch64 target this module compiles\n" + "for; these are pure register operations on values already in `uint32x4_t`.") + NEON_SAFETY64 = NEON_SAFETY.replace("uint32x4_t", "uint64x2_t") + def neon_body(helper, quad_ty, vocab, safety): + binds, lad = ladder(helper, "x", "y", "z", indent=" ", bind_indent=" ", **vocab) + return (" // GENERATED lowering (tools/gen_ternlog_bodies.py), per 128-bit quad (NEON).\n" + " Self(core::array::from_fn(|p| {\n" + " let (x, y, z) = (self.0[p].0, b.0[p].0, c.0[p].0);\n" + binds + "\n" + " // SAFETY: " + safety.replace("\n", "\n // ") + "\n" + f" {quad_ty}(unsafe {{\n" + lad + "\n })\n }))\n") + out["simd_neon.rs"] = ([ + ((fn_sig(r"impl U32x16 \{", " "), NEON_BODY_END), neon_body("ternlog_two_input_u32x4", "U32x4", NEONL, NEON_SAFETY)), + ((fn_sig(r"impl U64x8 \{", " "), NEON_BODY_END), neon_body("ternlog_two_input_u64x2", "U64x2", NEON64L, NEON_SAFETY64)), + ], "\n\n".join([ + two_input_fn("ternlog_two_input_u32x4", "uint32x4_t", NEON, "vdupq_n_u32(0)", "vdupq_n_u32(!0)", + attrs=('#[cfg(target_arch = "aarch64")]',), unsafe_reason=NEON_SAFETY), + two_input_fn("ternlog_two_input_u64x2", "uint64x2_t", NEON64, "vdupq_n_u64(0)", "vdupq_n_u64(!0)", + attrs=('#[cfg(target_arch = "aarch64")]',), unsafe_reason=NEON_SAFETY64), + ]), None) + worst[("simd_neon.rs", "U32x16")] = 7; worst[("simd_neon.rs", "U64x8")] = 8 # NOT via veor(x, all-ones) costs one more + # ── wasm: U32x16 and U64x8, both per-quad v128 through the one lane-agnostic helper ── + def wasm_body(quad_ty): + binds, lad = ladder("ternlog_two_input_v128", "x", "y", "z", indent=" ", bind_indent=" ", **WASML) + return (" // GENERATED lowering (tools/gen_ternlog_bodies.py), per 128-bit quad.\n" + " Self(core::array::from_fn(|p| {\n" + " let (x, y, z) = (self.0[p].0, b.0[p].0, c.0[p].0);\n" + binds + "\n" + f" {quad_ty}({{\n" + lad + "\n })\n }))\n") + out["simd_wasm.rs"] = ([ + ((fn_sig(r"impl U32x16 \{", " "), WASM_BODY_END), wasm_body("U32x4")), + ((fn_sig(r"impl U64x8 \{", " "), WASM_BODY_END), wasm_body("U64x2")), + ], two_input_fn("ternlog_two_input_v128", "v128", WASM, "u32x4_splat(0)", "u32x4_splat(!0)", indent=" "), + "pub mod wasm32_simd {") + worst[("simd_wasm.rs", "U32x16")] = 7; worst[("simd_wasm.rs", "U64x8")] = 7 + # Op-count assertion: the number the docs quote is measured on the emitted text. + for (fname, ty), bound in worst.items(): + reps, _h, _m = out[fname] + body = [n for ((sig, _e), n) in reps if f"impl {ty} " in sig.pattern.replace("\\{", "{").replace("\\", "")][0] + last_else = [l for l in body.splitlines() if "(t0, x, y)" in l or "(t0, self, b)" in l][-1] + ops = count_ops(last_else) + 2 * 2 + assert ops <= bound, (fname, ty, ops, bound) + if check: + # Compare AFTER rustfmt: the committed files are formatted, the emitted + # text is not (long ladder lines get wrapped), so raw bytes would + # always drift. Formatting is not content; a hand-edited arm still is. + import subprocess, tempfile, difflib + drift = [] + for fname, (reps, app, mod) in out.items(): + path = ROOT / fname + regenerated = apply(path, reps, app, inside_module=mod, write=False) + with tempfile.NamedTemporaryFile("w", suffix=".rs", delete=False, dir=str(ROOT.parent / "target") if (ROOT.parent / "target").exists() else None) as tf: + tf.write(regenerated); tmp = tf.name + subprocess.run(["rustfmt", "--edition", "2021", "--config-path", str(ROOT.parent), tmp], check=True) + formatted = pathlib.Path(tmp).read_text(); pathlib.Path(tmp).unlink() + if formatted != path.read_text(): + drift.append(fname) + for line in difflib.unified_diff(path.read_text().splitlines(), formatted.splitlines(), "committed", "regenerated", lineterm="", n=1): + print(line) + if drift: + print("DRIFT: generated bodies differ from the generator's output in:", ", ".join(drift)) + sys.exit(1) + print("check: all generated bodies current") + return + if not write: + for k, (reps, app, _m) in out.items(): + print(f"=== {k}\n{reps[0][1]}\n{app}\n") + return + for fname, (reps, app, mod) in out.items(): + apply(ROOT / fname, reps, app, inside_module=mod) + print("applied", fname) + +if __name__ == "__main__": + main("--apply" in sys.argv, check="--check" in sys.argv) diff --git a/tools/safe_intrinsic_probe/Cargo.toml b/tools/safe_intrinsic_probe/Cargo.toml new file mode 100644 index 00000000..e5be9573 --- /dev/null +++ b/tools/safe_intrinsic_probe/Cargo.toml @@ -0,0 +1,31 @@ +# safe_intrinsic_probe — which SIMD intrinsics are callable from SAFE code on +# the pinned toolchain, per architecture. Not a workspace member; run by hand +# after a toolchain bump (the answer is a toolchain property, not a code one): +# +# cd tools/safe_intrinsic_probe +# RUSTFLAGS="--cfg probe_a" cargo check --target aarch64-unknown-linux-gnu # E0133 on 1.98.1 +# RUSTFLAGS="--cfg probe_c" cargo check --target aarch64-unknown-linux-gnu # E0133 on 1.98.1 +# RUSTFLAGS="--cfg probe_a3 -Ctarget-cpu=x86-64-v4" cargo check # E0133 on 1.98.1 +# RUSTFLAGS="--cfg probe_a -Ctarget-feature=+simd128" cargo check --target wasm32-unknown-unknown # OK +# +# Result matrix and the consequence for the backends: see +# .claude/knowledge/vertical-simd-consumer-contract.md § "unsafe at the +# intrinsic boundary". Uses the repo's rust-toolchain.toml via the symlink. +# Standalone: opt out of the parent workspace (a manifest below a workspace root +# that is neither a member nor excluded makes cargo refuse to build it — Codex +# P2 on PR #306). +[workspace] + +[package] +name = "safe_intrinsic_probe" +version = "0.0.0" +edition = "2021" +publish = false + +[lib] +path = "src/lib.rs" + +# The probe arms are opt-in cfgs, not features: declare them so rustc's +# `unexpected_cfgs` check knows they are deliberate. +[lints.rust] +unexpected_cfgs = { level = "warn", check-cfg = ['cfg(probe_a)', 'cfg(probe_a2)', 'cfg(probe_a3)', 'cfg(probe_c)'] } diff --git a/tools/safe_intrinsic_probe/rust-toolchain.toml b/tools/safe_intrinsic_probe/rust-toolchain.toml new file mode 120000 index 00000000..e01fe10a --- /dev/null +++ b/tools/safe_intrinsic_probe/rust-toolchain.toml @@ -0,0 +1 @@ +../../rust-toolchain.toml \ No newline at end of file diff --git a/tools/safe_intrinsic_probe/src/lib.rs b/tools/safe_intrinsic_probe/src/lib.rs new file mode 100644 index 00000000..489b2e4d --- /dev/null +++ b/tools/safe_intrinsic_probe/src/lib.rs @@ -0,0 +1,47 @@ +#![forbid(unsafe_code)] +//! Which SIMD intrinsics are callable from SAFE code on rustc 1.98.1, per arch. +//! +//! Nothing here is an API and nothing calls these functions: the OBSERVABLE +//! is whether `cargo check` accepts or rejects each arm (E0133), so they are +//! crate-private and the dead-code lint is silenced rather than satisfied by +//! fake callers or doc examples that would have to be `#[cfg]`-gated per arch. +#![allow(dead_code)] +#[cfg(target_arch = "aarch64")] +pub mod a64 { + use core::arch::aarch64::*; + /// A: plain fn, baseline-feature intrinsic (neon). + #[cfg(probe_a)] + pub(crate) fn plain_neon(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { vandq_u32(a, b) } + /// B: annotated fn, same call. + #[target_feature(enable = "neon")] + pub(crate) fn annotated_neon(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { vandq_u32(a, b) } + /// C: plain fn calling the annotated SAFE fn. + #[cfg(probe_c)] + pub(crate) fn plain_calls_annotated(a: uint32x4_t, b: uint32x4_t) -> uint32x4_t { annotated_neon(a, b) } +} +#[cfg(target_arch = "x86_64")] +pub mod x86 { + use core::arch::x86_64::*; + /// A: plain fn, sse2 (baseline for x86_64). + #[cfg(probe_a)] + pub(crate) fn plain_sse2(a: __m128i, b: __m128i) -> __m128i { _mm_and_si128(a, b) } + /// A2: plain fn, avx2 (baseline only under -Ctarget-cpu=x86-64-v3). + #[cfg(probe_a2)] + pub(crate) fn plain_avx2(a: __m256i, b: __m256i) -> __m256i { _mm256_and_si256(a, b) } + /// A3: plain fn, avx512f ternarylogic (baseline only under x86-64-v4). + #[cfg(probe_a3)] + pub(crate) fn plain_avx512(a: __m512i, b: __m512i, c: __m512i) -> __m512i { _mm512_ternarylogic_epi64::<0x96>(a, b, c) } + /// B: annotated fn. + #[target_feature(enable = "avx2")] + pub(crate) fn annotated_avx2(a: __m256i, b: __m256i) -> __m256i { _mm256_and_si256(a, b) } + /// C: plain fn calling annotated safe fn. + #[cfg(probe_c)] + pub(crate) fn plain_calls_annotated(a: __m256i, b: __m256i) -> __m256i { annotated_avx2(a, b) } +} +#[cfg(target_arch = "wasm32")] +pub mod w { + use core::arch::wasm32::*; + /// A: plain fn, simd128 intrinsic. + #[cfg(probe_a)] + pub(crate) fn plain_wasm(a: v128, b: v128) -> v128 { v128_and(a, b) } +}