diff --git a/.cargo/config-v3.toml b/.cargo/config-v3.toml new file mode 100644 index 00000000..32a16e1e --- /dev/null +++ b/.cargo/config-v3.toml @@ -0,0 +1,32 @@ +# x86-64-v3 (AVX2) — the PORTABLE DISTRIBUTION BASELINE, pinned explicitly. +# +# Usage: +# env -u RUSTFLAGS cargo --config .cargo/config-v3.toml +# CARGO_ARGS='--config .cargo/config-v3.toml' bash scripts/masking-parity.sh native +# +# v3 is portable across all x86_64 silicon shipping since ~2013 (Haswell+) and +# is what a general-distribution build should target. Until 2026-09-16 it was +# `.cargo/config.toml`'s default and therefore UNNAMED at every call site; the +# default is now `target-cpu=native` (see that file's superseded-in-place note +# for why), so the tier a row depends on is named BY that row. This file is +# where "portable" is spelled. +# +# Overlay semantics, measured 2026-09-16: cargo JOINS `target..rustflags` +# across config files and the LAST `-Ctarget-cpu` wins, so this file needs the +# target-cpu ONLY — `.cargo/config.toml`'s two crypto-backend cfgs +# (`curve25519_dalek_backend="serial"`, `poly1305_force_soft`) come through the +# join and stay in force. Verified with `cargo build -p encryption -v`: both +# `-Ctarget-cpu` values present, v3 last, both cfgs present. +# +# `env -u RUSTFLAGS` is load-bearing: a RUSTFLAGS env REPLACES every +# cargo-config rustflags entry, so with one set this file does not apply AND +# neither do the crypto cfgs. +# +# No `-Dwarnings` here, deliberately — unlike `config-v4.toml`. A config that +# promotes warnings to errors turns a disable-run (which typically orphans a +# binding) into "did not compile", which reads identically to "the guard was +# not load-bearing" when piped through a grep. The portable arm is the one most +# likely to be used for a disable-run, so it stays warnings-permissive; the +# repo's `-D warnings` gate is the explicit clippy invocation. +[target.'cfg(target_arch = "x86_64")'] +rustflags = ["-Ctarget-cpu=x86-64-v3"] diff --git a/.cargo/config.toml b/.cargo/config.toml index cd2a62da..2aeb3e61 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,22 +1,64 @@ [build] -# Default cargo config — x86-64-v3 (AVX2) baseline. Portable across all -# x86_64 silicon shipping since ~2013 (Haswell+). This is what GitHub CI -# runs against and what `cargo build` produces for general distribution. +# Default cargo config — `target-cpu=native`: the default MEASURES THE MACHINE +# IT RUNS ON. # -# Why v3 and not "no target-cpu": +# ### ⊘ SUPERSEDED 2026-09-16: this file used to pin x86-64-v3 as the default +# +# The v3 pin was correct about one thing and wrong about the consequence. It IS +# the portable distribution baseline — that part stands and is now spelled +# explicitly in `.cargo/config-v3.toml`. What it got wrong is that a default +# naming a tier the host is NOT means every AVX-512 measurement needs an +# incantation, and a forgotten incantation does not fail — it silently grades +# the wrong tier. Measured the day of the flip: `scripts/codegen-witness.sh +# avx512` run WITHOUT `CARGO_ARGS='--config .cargo/config-v4.toml'` built v3 and +# reported three FAILs ("has no vpternlog on an AVX-512 build") on probe symbols +# the change under test never touched. The assertion was right, the build was +# the wrong one, and nothing in the output said so. +# +# `native` cannot mis-grade that way: rustc resolves the host CPUID, so the +# default arm is always the arm this machine can actually run, and it can never +# SIGILL by construction. The cost is stated plainly rather than hidden: the +# default no longer NAMES a tier, so **a measurement is labelled by the arm the +# program itself reports** (`simd-masking-parity` and every probe print +# `avx512f=true|false`), never by "it was the default". A tier a row depends on +# is pinned by that row, explicitly: +# +# portable / distribution baseline --config .cargo/config-v3.toml (v3, AVX2) +# ^ REQUIRED for anything you ship. A plain +# `cargo build --release` is now tuned to the BUILD HOST and is not portable +# — `.cargo/config-native.toml` has always said so about this exact flag +# ("do NOT distribute artifacts built with this config"), and that warning now +# applies to the default. Runtime `simd_caps()` dispatch does NOT rescue it: +# detection picks among code paths, it cannot un-emit host-only instructions +# the baseline codegen already placed everywhere. (Raised by codex on #313; +# README's build table and both Dockerfiles were corrected in the same PR.) +# Nothing is shipped from a default build today — this crate is a `[lib]` with +# no bin targets, and the published crate does not carry this file to +# consumers, who build under their own config. The caveat is for THIS +# workspace's own release artifacts and images. +# AVX-512 --config .cargo/config-v4.toml (v4) +# Sapphire Rapids (VNNI/BF16/AMX) --config .cargo/config-avx512.toml +# +# This does NOT reach `.github/workflows/ci.yaml`. That workflow sets a global +# `RUSTFLAGS: "-D warnings"`, and a RUSTFLAGS env REPLACES every cargo-config +# rustflags entry — so none of the flags in this file have ever applied there, +# including the two crypto cfgs below. Measured two-sided on the same unit the +# same day: no RUSTFLAGS → 65× `-Ctarget-cpu`, 65× `poly1305_force_soft`; +# `RUSTFLAGS="-D warnings"` → zero of each. +# +# Why NOT "no target-cpu at all": # `src/simd_avx2.rs` composes `F32x16` as two `__m256` halves (AVX # intrinsics), and the `simd_avx2_*` op funcs use `__m256i` (AVX2). -# Without a global v3 baseline, rustc compiles to x86-64 generic (SSE2) -# and those intrinsics emit instructions the CPU never executes → -# SIGILL at run time, exactly the PR #170 CI failure mode. -# -# AVX-512 builds: use `--config .cargo/config-avx512.toml` (or -# `CARGO_BUILD_RUSTFLAGS='-Ctarget-cpu=x86-64-v4'`). The simd.rs dispatch -# arms key off `target_feature = "avx512f"`; under v4 they pick the -# `simd_avx512` backend (native `__m512` / `__m512d` / `__m512i`). +# With NO baseline at all, rustc compiles to x86-64 generic (SSE2) and +# those intrinsics emit instructions the CPU never executes → SIGILL at +# run time, exactly the PR #170 CI failure mode. `native` clears that floor +# on any host that has the features, and on a host that does not, the +# backend `simd.rs` selects is the one that host can run. # -# Build-machine-tuned binaries: use `--config .cargo/config-native.toml` -# (`target-cpu = "native"`); rustc resolves the host CPUID at compile. +# `.cargo/config-native.toml` is kept and is now a no-op overlay on x86_64 +# (same flag as the default). It stays because it names the intent at a call +# site and because a caller may pass it on a host whose own default config +# differs. # # Runtime LazyLock dispatch (one release binary, heterogeneous deployment # silicon) is a fifth opt-in mode — see § 7.1 of @@ -80,7 +122,7 @@ # Verify: cargo build -p encryption -v 2>&1 | grep poly1305_force_soft [target.'cfg(target_arch = "x86_64")'] rustflags = [ - "-Ctarget-cpu=x86-64-v3", + "-Ctarget-cpu=native", "--cfg", "curve25519_dalek_backend=\"serial\"", "--cfg", diff --git a/.claude/blackboard.md b/.claude/blackboard.md index 0845faf8..b0318ed9 100644 --- a/.claude/blackboard.md +++ b/.claude/blackboard.md @@ -1,3 +1,173 @@ +## 2026-09-16 (14) — the default `target-cpu` now MEASURES THE HOST; a config that a caller can silently REPLACE was never a guarantee + +Three findings, one root cause, PR #313 (branch `claude/c64-6502-falsifier-shztkk`). +The root cause is one sentence: **a build flag you did not have to ask for is a +flag you cannot tell you got.** + +### 1. The flip: `.cargo/config.toml` → `target-cpu=native` + +It used to pin `x86-64-v3`. That is the correct PORTABLE baseline and still is — +it moved into `.cargo/config-v3.toml`, where a row that needs it SAYS so. What +was wrong was making it the *unnamed default*, because then every AVX-512 +measurement needs an incantation, and a forgotten incantation does not fail. + +**The incident that prompted it, same session.** Gating the `pack` fold +I ran `scripts/codegen-witness.sh avx512` WITHOUT +`CARGO_ARGS='--config .cargo/config-v4.toml'`. It built v3 and printed: + +``` + probe_ternlog_u64x8: 0 vpternlog + FAIL: probe_ternlog_u64x8 has no vpternlog on an AVX-512 build +``` + +Three FAILs, on probe symbols the change under test never touched. The assertion +was right, the build was the wrong one, and **nothing in the output said which**. +That is the workspace's own recorded trap — *a timing without its target-cpu is +an anecdote* — with a second door: an ASSERTION without its target-cpu is a +false alarm, and a false alarm on symbols you did not touch is the shape most +likely to be believed. + +After the flip the identical bare command PASSES with 6 `vpternlog`. + +### 2. The pin is load-bearing in BOTH directions — measured two-sided + +A flip that only made v4 easier would have moved the landmine, not removed it. +On this AVX-512 host: + +| command | result | +|---|---| +| `codegen-witness.sh avx2` bare | **FAIL** `has no packed logic` — grading v4 assembly against "no vpternlog may appear" | +| `codegen-witness.sh avx2` + `config-v3` | **PASS** | + +So the matrix's portable row now PINS v3 instead of inheriting it. Unpinned it +would grade whichever tier the runner SKU happens to be — and some Azure runner +generations carry AVX-512, so it would be **nondeterministic across reruns**, +which is worse than wrong. + +**The rule this generalizes to: a tier is READ, never inferred.** Every probe +and the parity program print `avx512f=true|false` precisely because the default +no longer names a tier. Cite that line, not "it was the default". + +### 3. The bigger find, and it was NOT the one I went looking for + +`.github/workflows/ci.yaml` sets a workflow-global `RUSTFLAGS: "-D warnings"`. +A RUSTFLAGS env **REPLACES** every cargo-config `rustflags` entry rather than +joining it. So from the moment that variable was introduced, **nothing in +`.cargo/config.toml` had ever applied to any job in that workflow** — not the +target-cpu, and not the two cfgs that compile out curve25519-dalek's AVX2 +backend (57 raw `_mm*` under 52 `unsafe`) and poly1305's (424 under 30). + +Those are the second and third unaudited SIMD surfaces beside `ndarray::simd`, +in the crypto path. The config file argues at length for keeping them out of the +binary — the matryoshka rule. **In CI they were in.** + +Measured two-sided, same tree, same unit (`cargo build -p encryption -v`): + +| RUSTFLAGS | `-Ctarget-cpu` | `poly1305_force_soft` | +|---|---:|---:| +| unset | 65× | 65× | +| `-D warnings` | **0** | **0** | +| `-D warnings` + the two cfgs | — | present, clean | + +Fixed by putting the two **arch-neutral** cfgs into that global RUSTFLAGS. +`-Ctarget-cpu` stays out for the reason it was removed (i686 is 32-bit, s390x is +not x86); both cfgs are read by their crates on every arch. + +**⊘ Correction to how this file has been reasoning.** Several earlier entries +cite `.cargo/config.toml:83` as evidence of what a build *was*. That is sound +only when no RUSTFLAGS env is set. **Before citing any config flag as in force, +check whether the caller sets RUSTFLAGS.** Entries (13) and earlier are not +wrong — they used `env -u RUSTFLAGS` — but the habit of citing the file rather +than the arm's own report is what let this sit unnoticed. + +### 4. Two real defects the flip surfaced on DAY ONE + +Both in code the v3 default never compiled, and therefore never linted: + +- `src/simd_int_ops.rs` — `needless_return` in the runtime-VNNI block. The lint + is **config-dependent**: the trailing scalar fallback is cfg'd out when + `avx512vnni`/`avxvnni` is a compile feature, so the second `return` is + trailing there and load-bearing on v3. **`allow`, not `expect`** — `expect` + would fail the v3 build for the lint NOT firing, turning one arm's cleanup + into the other arm's error. Worth keeping: a cfg-dependent lint is the one + case where `expect` is the wrong tool. +- `examples/ternlogq_tail_descent_probe.rs` — `print_literal`. Gated + `avx512f + avx512vl`, so #311's own clippy run never compiled it. My #311 + commit message claimed "clippy v3 `--examples --tests` clean"; that was TRUE + and did not cover this file. **A green lint over code that was cfg'd out is + not evidence about that code.** + +### 5. The open question — ANSWERED the same day, and the answer is bigger + +A new `host-native` matrix row runs the unpinned parity program and is +**`continue-on-error` on purpose**: a row whose answer is "whatever this +runner is" cannot gate a merge on pool scheduling. + +**Measured on its first run, and it beat the question.** Within ONE workflow +run (35148155422, head `c1bd7015`), two jobs — both `runs-on: ubuntu-latest`, +both under the `target-cpu=native` default — reported different tiers: + +| job | reports | +|---|---| +| `realization/nightly x x86_64` | `avx512f=TRUE` | +| `realization/host-native x x86_64` | `avx512f=FALSE` | + +**GitHub's `ubuntu-latest` pool is HETEROGENEOUS: the tier is decided per +JOB, not per run and not per repo.** So `native` in CI is a coin flip, and +an ISA assertion on an unpinned row would pass or fail on scheduling. That +is the empirical vindication of pinning the portable row — a green unpinned +run would have proven only that the day's scheduling was lucky. + +**⊘ Correction to this session's own reasoning, recorded because the error is +instructive.** When the nightly row failed I inferred "the GitHub runner has +AVX-512" from the failure's mechanism alone (the errors sat in +`#[cfg(all(test, target_feature = "avx512f"))]` modules, so that predicate +had to be true). The inference was locally valid and the generalization was +wrong: it was true of THAT job, and false of another job in the same run. **A +mechanism that proves a fact about one runner proves nothing about "the +runner".** The `host-native` row is what caught it, which is the whole reason +a row that only reports is worth having. + +### 5b. What the nightly CI failure actually was — a REAL bug, not collateral + +`realization/nightly x x86_64` went red on the first push. Root cause, and it +is the flip earning its keep rather than the flip breaking something: + +`cargo +nightly test --features nightly-simd` **fails to compile on ANY host +where `avx512f` is a compile-time feature**, and has for as long as both +existed. The call sites live in `#[cfg(all(test, target_feature = "avx512f"))]` +modules of `src/simd_avx512.rs`; under the old v3 default that predicate was +false, so the two features never co-compiled anywhere — not in CI, not +locally. Any developer on an AVX-512 machine hits it today. + +The gap was a stated-contract violation: both polyfill files' own doc comments +say *"API mirrors `simd_avx512::` so consumer code is backend-agnostic"*, +and four types were short — `I8x64`/`I8x32` (zero, add, sub, cmp_gt) and +`I16x32`/`I16x16` (those plus min, max). + +**Fixed the surface, did not pin the row.** Pinning the nightly row to v3 +would have hidden a defect that bites outside CI and stopped that row ever +witnessing the combination again — "disable the thing that found the bug". + +Semantics were READ off the native bodies, not guessed: `add`/`sub` are +`_mm512_add/sub_epi{8,16}`, i.e. WRAPPING, so the polyfill uses `+`/`-` and +NOT the `saturating_*` methods sitting next to them, which are a different +operation and the obvious way to get this subtly wrong. `cmp_gt` delegates to +each type's existing `cmpgt_mask` so the two spellings cannot drift. + +Evidence is a RUN, not a lint: `cargo +nightly test --lib --features +nightly-simd` -> **2534 passed**, and the ones that matter are the AVX-512 +backend's OWN test vectors now executing against the `core::simd` polyfill and +agreeing with the native expectations. That is cross-backend parity this repo +did not previously have. + +### What did NOT change + +v3 is still the portable distribution baseline. The SIGILL floor argument in the +config file still holds (`simd_avx2.rs`'s `__m256`/`__m256i` bodies need AVX2 +present) — `native` clears it on any host that has the features, and on a host +that does not, `simd.rs` selects a backend that host can run. + ## 2026-09-16 (13) — the `VPTERNLOGQ` tail is a DESCENT, not a pad (5–8×); a 64×2 re-apply on a full-width mask is NOT (0.5–0.7×); the GEMM block-stop tail is INERT (0.99–1.02×) Three probes, one question in three places (operator: *"instead of padding the diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 6a6436dc..cc98a65d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -20,7 +20,32 @@ env: # `.cargo/config.toml`: per-function `#[target_feature]` + runtime # `LazyLock` detection means one binary, all ISAs. Jobs that # specifically need a higher target-cpu can opt in via per-job env. - RUSTFLAGS: "-D warnings" + # + # The two `--cfg` flags are NOT a preference — they are the matryoshka + # guarantee, and they are here because SETTING THIS VARIABLE AT ALL SILENTLY + # DISABLED THEM (found 2026-09-16). A RUSTFLAGS env REPLACES every + # cargo-config `rustflags` entry rather than joining it, so from the moment + # this workflow gained a global RUSTFLAGS, NOTHING in `.cargo/config.toml` + # applied to any job here — including the two cfgs that compile out + # curve25519-dalek's AVX2 backend (57 raw `_mm*` intrinsics under 52 + # `unsafe`) and poly1305's (424 under 30). Those are second and third + # unaudited SIMD surfaces beside `ndarray::simd`, in the crypto path, and + # `.cargo/config.toml` argues at length for keeping them out. In CI they + # were in. + # + # Measured two-sided on one unit, same tree, same day: + # no RUSTFLAGS env -> 65x -Ctarget-cpu, 65x poly1305_force_soft + # RUSTFLAGS="-D warnings" -> ZERO of each + # RUSTFLAGS="-D warnings " -> both cfgs present, build clean + # + # Only the ARCH-NEUTRAL half is restored here. `-Ctarget-cpu` stays out for + # the reason above (i686 / s390x); both cfgs are read by their crates on + # every arch, so they are safe across the whole cross matrix. + # + # Rule for anyone editing this line: a flag added to a RUSTFLAGS env does not + # ADD to the cargo config, it REPLACES it. Re-read `.cargo/config.toml` + # before changing this value and carry forward anything still needed. + RUSTFLAGS: '-D warnings --cfg curve25519_dalek_backend="serial" --cfg poly1305_force_soft' MSRV: 1.98.1 BLAS_MSRV: 1.98.1 diff --git a/.github/workflows/simd-matrix.yaml b/.github/workflows/simd-matrix.yaml index 09535a0e..a2cc781c 100644 --- a/.github/workflows/simd-matrix.yaml +++ b/.github/workflows/simd-matrix.yaml @@ -67,8 +67,21 @@ env: jobs: native: - # x86_64 host at the crate's default target-cpu (x86-64-v3 = the AVX2 - # realization). Also the ONLY row that can exercise AMX: the tile ops are + # x86_64 at x86-64-v3 (the AVX2 realization), PINNED EXPLICITLY via + # `.cargo/config-v3.toml`. + # + # ⊘ This row used to rely on v3 being `.cargo/config.toml`'s DEFAULT. That + # default is now `target-cpu=native`, so the pin is load-bearing rather + # than decorative, and the reason is two-sided-measured (2026-09-16, on an + # AVX-512 host): `codegen-witness.sh avx2` BARE reports + # "FAIL: ... has no packed logic" — it is grading vpternlog-carrying v4 + # assembly against an assertion that says no vpternlog may appear — while + # the same command with `CARGO_ARGS='--config .cargo/config-v3.toml'` + # PASSES. Unpinned, this row would grade whichever tier the runner SKU + # happens to be; some Azure runner generations carry AVX-512, so it would + # be nondeterministic across reruns, not merely wrong. + # + # Also the ONLY row that can exercise AMX: the tile ops are # runtime-gated and always compiled into native builds, so the report # prints which gates this runner clears (`tile_available`/`available` # false on a non-AMX runner is the expected, honest answer) and the @@ -83,15 +96,70 @@ jobs: - uses: Swatinem/rust-cache@v2 - name: generated ternlog bodies are current run: python3 tools/gen_ternlog_bodies.py --check - - name: masking parity (native, default config) - run: bash scripts/masking-parity.sh native - - name: codegen witness (avx2) - run: bash scripts/codegen-witness.sh avx2 + - name: masking parity (x86-64-v3, pinned) + run: CARGO_ARGS='--config .cargo/config-v3.toml' bash scripts/masking-parity.sh native + - name: codegen witness (avx2, pinned to v3) + run: CARGO_ARGS='--config .cargo/config-v3.toml' bash scripts/codegen-witness.sh avx2 - name: AMX realization report (runtime-gated; prints this runner's gates) run: cargo run --example amx_realization_report - name: AMX encoding + detection tests (no tile op executes) run: cargo test --lib -- hpc::amx_ops simd_amx + host-native: + # INFORMATIONAL, non-gating (`continue-on-error`). Testing the waters: + # what does a GitHub runner actually GIVE us under the new + # `target-cpu=native` default? + # + # Every other row in this matrix names its tier and asserts against it. + # This one names NOTHING and just reports — because the open question the + # native default raises is empirical and we do not have the answer: + # GitHub's ubuntu-latest pool is not one SKU, and some generations carry + # AVX-512 while others do not. The parity program's own header line + # (`avx512f=true|false`) is the reading; `lscpu` beside it is the + # corroboration. + # + # ANSWER, measured 2026-09-16 on this row's first run — and it is stronger + # than the question asked. Within ONE workflow run (35148155422), two jobs + # both `runs-on: ubuntu-latest`, both under the `target-cpu=native` + # default, reported DIFFERENT tiers: + # + # realization/nightly x x86_64 avx512f=TRUE + # realization/host-native x x86_64 avx512f=FALSE + # + # So the pool is HETEROGENEOUS and the tier is decided per JOB, not per + # run and not per repo. (The nightly row's failure on the previous head + # was caused by landing on an AVX-512 runner, which compiled + # `#[cfg(all(test, target_feature = "avx512f"))]` modules that had never + # been compiled in CI before — a real polyfill gap, fixed in c1bd7015.) + # + # That is exactly why `native` pins nothing and the portable row pins v3: + # unpinned, an ISA assertion here would be a coin flip per job, and a + # green run would prove only that today's scheduling was lucky. + # + # It is `continue-on-error` ON PURPOSE and must stay that way: a row whose + # result is "whatever this runner is" cannot gate a merge without making + # the merge depend on pool scheduling. If a future session wants to ASSERT + # a tier here, that is a different row with an explicit pin — do not + # promote this one by deleting the flag. + # + # What it can still catch, and why it is worth a row at all: the parity + # program must be bit-identical to its scalar references on WHATEVER + # realization it lands on. A red here is a real parity failure on a tier + # no pinned row happens to cover. + runs-on: ubuntu-latest + name: realization/host-native × x86_64 (informational) + continue-on-error: true + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - name: what silicon is this runner + run: lscpu | sed -n '1,/^Flags/p' | head -30 + - name: masking parity (host-native, unpinned — READ THE HEADER LINE) + run: bash scripts/masking-parity.sh native + native-v4: # Same host, AVX-512 realization via the v4 cargo config. Building and # inspecting the assembly needs no AVX-512 silicon; RUNNING the parity diff --git a/CLAUDE.md b/CLAUDE.md index 51c0dd52..62ae0a2c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -104,21 +104,61 @@ src/ ### New Modules - `src/hpc/styles/` — 34 cognitive primitives (rte, htd, smad, tcp, irs, mcp, tca, cdt, mct, lsi, pso, cdi, cws, are, tcf, ssr, etd, amp, zcf, hpm, cur, mpc, ssam, idr, spp, icr, sdd, dtmf, hkf). Each is `fn(Base17, NarsTruth) → result`. 49 tests. - `src/hpc/causal_diff.rs` — CausalEdge64 (u64 packed), scaffold_to_palette3d_layers(), quality scoring (GOOD/BAD/UNCERTAIN), NARS self-reinforcement LoRA, PAL8 serialization (4101 bytes). -- **Build config — AVX-512 is NOT the default, and believing it is corrupts - measurements.** `.cargo/config.toml` sets **`x86-64-v3` (AVX2)**, deliberately: - it is the portable CI/distribution baseline, and its own comment explains why - (without a v3 floor the AVX2 intrinsics in `simd_avx2.rs` SIGILL). A plain - `cargo build`/`run`/`test` therefore measures **v3**. - **For AVX-512 you must ask for it, every time:** +- **Build config — the default is `target-cpu=native`: it MEASURES THE MACHINE + IT RUNS ON, and therefore NAMES NO TIER.** + + > ⊘ **SUPERSEDED 2026-09-16.** This bullet read *"AVX-512 is NOT the default… + > `.cargo/config.toml` sets `x86-64-v3` (AVX2), deliberately… for AVX-512 you + > must ask for it, every time."* That was accurate for the old default and is + > now wrong on its main clause. **What survives unchanged: v3 IS the portable + > distribution baseline** — it moved out of the unnamed default into + > `.cargo/config-v3.toml`, where a row that depends on it says so. + > + > Why the flip: a default naming a tier the host is not means every AVX-512 + > measurement needs an incantation, and a forgotten incantation does not fail + > — it grades the wrong tier silently. Measured the day of the flip: + > `scripts/codegen-witness.sh avx512` run WITHOUT + > `CARGO_ARGS='--config .cargo/config-v4.toml'` built v3 and reported three + > `FAIL: … has no vpternlog on an AVX-512 build` on probe symbols the change + > under test never touched. Same command after the flip: PASS, 6 vpternlog. + + A plain `cargo build`/`run`/`test` therefore measures **whatever this host + is** — on an AVX-512 box, AVX-512. **So a tier is never inferred from "it was + the default"; it is READ from the arm's own report** (`simd-masking-parity` + and every probe print `avx512f=true|false`) or PINNED by the caller: ```sh + env -u RUSTFLAGS cargo --config .cargo/config-v3.toml # portable baseline (AVX2) env -u RUSTFLAGS cargo --config .cargo/config-v4.toml run --release --example ``` + **The pin is load-bearing in BOTH directions, measured two-sided the same + day on an AVX-512 host:** `codegen-witness.sh avx2` BARE now FAILS + (`has no packed logic` — it is grading v4 assembly against an assertion that + no vpternlog may appear), and PASSES with + `CARGO_ARGS='--config .cargo/config-v3.toml'`. The portable CI row pins it + for exactly this reason. + `env -u RUSTFLAGS` is load-bearing: a RUSTFLAGS env var REPLACES every cargo-config rustflags entry, so it silently drops `-Ctarget-cpu=x86-64-v4` and the arm measures v3 while claiming v4 (the trap `scripts/masking-parity.sh` - documents). Verify the arm you got — the parity program prints + documents). + + **That same mechanism had silently disabled the whole config in CI, and it is + the more serious half (found 2026-09-16).** `.github/workflows/ci.yaml` sets a + workflow-global `RUSTFLAGS: "-D warnings"`, so **none** of + `.cargo/config.toml`'s flags had ever applied to any job in that workflow — + not the target-cpu, and **not the two crypto-backend cfgs** + (`curve25519_dalek_backend="serial"`, `poly1305_force_soft`) that keep + dalek's 57 and poly1305's 424 raw intrinsics OUT of the binary. That is the + matryoshka guarantee the config file argues for at length, absent in CI. + Measured two-sided on one unit: no RUSTFLAGS → 65× `-Ctarget-cpu`, 65× + `poly1305_force_soft`; `RUSTFLAGS="-D warnings"` → **zero of each**. Fixed by + putting the two **arch-neutral** cfgs into that global RUSTFLAGS (target-cpu + stays out — it is the arch-sensitive part, correctly removed for the i686 / + s390x cross rows). **The general rule: a config that can be silently replaced + is not a guarantee.** Before citing any `.cargo/config.toml` flag as being in + force, check whether the caller sets RUSTFLAGS. Verify the arm you got — the parity program prints `avx512f=true|false`, and any probe that reports timings should too. `.cargo/config-avx512.toml` is the stricter Sapphire Rapids EXECUTION config (VNNI/BF16/FP16/AMX) and SIGILLs on earlier AVX-512 silicon; `config-native.toml` @@ -161,9 +201,25 @@ and the dynamic `qemu-aarch64` from `qemu-user` leaves a second, differently worded failure (`command not found`) that looks like a fresh problem rather than the same one. Install `qemu-user-static`. -**And `native` is the AVX2 arm, not the AVX-512 one** — it takes -`.cargo/config.toml` (v3), so a green `native` leaves every `_mm512_*` body -unwitnessed. The AVX-512 arm is the binary run under the v4 config directly: +**And `native` is the HOST arm — it names no tier at all.** + +> ⊘ **SUPERSEDED 2026-09-16.** This line read *"`native` is the AVX2 arm, not +> the AVX-512 one — it takes `.cargo/config.toml` (v3)"*. True until the default +> flipped; false the moment it did, and it is the SAME defect class this whole +> section warns about — a doc asserting a tier instead of reading the arm's own +> report. Caught in review, on the PR that caused it. + +`scripts/masking-parity.sh native` builds with the DEFAULT config, which is now +`target-cpu=native`: on an AVX-512 host that arm is AVX-512, on a v3 host it is +AVX2. So a green `native` witnesses **whatever this machine is** — read the +header line to find out which. To witness AVX2 specifically, pin it: + +```sh +CARGO_ARGS='--config .cargo/config-v3.toml' bash scripts/masking-parity.sh native +``` + +The AVX-512 arm can also be pinned explicitly, which is what CI does and what +you want when the host is not AVX-512: ```sh cd crates/simd-masking-parity diff --git a/Dockerfile b/Dockerfile index f4fcf29b..08e979aa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -72,16 +72,27 @@ COPY ndarray-rand/benches/ ndarray-rand/benches/ # detects AVX-512 at runtime via LazyLock even when compiled for v3; # compile-time v3 just means the scalar/AVX2 fallback paths are used when the # runtime check fails. Both paths produce identical results. -ENV RUSTFLAGS="-C target-cpu=x86-64-v3" +# The tier is passed as a CONFIG, not as `ENV RUSTFLAGS` (changed 2026-09-16). +# A RUSTFLAGS env REPLACES every cargo-config `rustflags` entry rather than +# joining it, so `ENV RUSTFLAGS="-C target-cpu=x86-64-v3"` did set the tier — +# and silently dropped `.cargo/config.toml`'s two crypto-backend cfgs +# (`curve25519_dalek_backend="serial"`, `poly1305_force_soft`) that compile out +# curve25519-dalek's and poly1305's raw-intrinsic AVX2 backends. This image +# therefore shipped the unaudited SIMD surfaces the matryoshka rule exists to +# keep out. `--config` JOINS, so the tier AND the cfgs both apply. +# +# Passing it explicitly is also now required rather than optional: the default +# `.cargo/config.toml` is `target-cpu=native`, which tunes the artifact to +# whatever machine built the image and is not portable. # Build default features -RUN cargo build --release 2>&1 && echo "=== DEFAULT BUILD OK ===" +RUN cargo --config .cargo/config-v3.toml build --release 2>&1 && echo "=== DEFAULT BUILD OK ===" # Build with JIT -RUN cargo build --release --features jit-native 2>&1 && echo "=== JIT-NATIVE BUILD OK ===" +RUN cargo --config .cargo/config-v3.toml build --release --features jit-native 2>&1 && echo "=== JIT-NATIVE BUILD OK ===" # Run tests -RUN cargo test --release --lib -- hpc:: 2>&1 && echo "=== HPC TESTS OK ===" +RUN cargo --config .cargo/config-v3.toml test --release --lib -- hpc:: 2>&1 && echo "=== HPC TESTS OK ===" # Minimal runtime image — just proves it compiled FROM debian:bookworm-slim diff --git a/Dockerfile.avx512 b/Dockerfile.avx512 index 3cc76f9c..21d2e1e9 100644 --- a/Dockerfile.avx512 +++ b/Dockerfile.avx512 @@ -50,12 +50,19 @@ COPY examples/ examples/ COPY benches/ benches/ COPY ndarray-rand/benches/ ndarray-rand/benches/ -# AVX-512 pinned: compile-time dispatch, everything inlined -ENV RUSTFLAGS="-C target-cpu=x86-64-v4" +# AVX-512 pinned: compile-time dispatch, everything inlined. +# +# Passed as a CONFIG, not `ENV RUSTFLAGS` (changed 2026-09-16): a RUSTFLAGS env +# REPLACES every cargo-config `rustflags` entry, so the old form set the tier +# and silently dropped `.cargo/config.toml`'s two crypto-backend cfgs, shipping +# curve25519-dalek's and poly1305's raw-intrinsic AVX2 backends into the image. +# `--config` joins, so tier + cfgs both apply. (`config-v4.toml` also carries +# `-Dwarnings`, which is fine for an image build and is why these RUNs must +# stay warnings-clean.) -RUN cargo build --release 2>&1 && echo "=== AVX-512 BUILD OK ===" -RUN cargo build --release --features jit-native 2>&1 && echo "=== AVX-512 JIT BUILD OK ===" -RUN cargo test --release --lib -- hpc:: 2>&1 && echo "=== AVX-512 HPC TESTS OK ===" +RUN cargo --config .cargo/config-v4.toml build --release 2>&1 && echo "=== AVX-512 BUILD OK ===" +RUN cargo --config .cargo/config-v4.toml build --release --features jit-native 2>&1 && echo "=== AVX-512 JIT BUILD OK ===" +RUN cargo --config .cargo/config-v4.toml test --release --lib -- hpc:: 2>&1 && echo "=== AVX-512 HPC TESTS OK ===" FROM debian:bookworm-slim COPY --from=builder /app/target/release/libndarray.rlib /usr/local/lib/ diff --git a/README.md b/README.md index eb9a6d17..3e6aeb4e 100644 --- a/README.md +++ b/README.md @@ -198,14 +198,22 @@ if caps.neon { println!("ARM profile: {}", caps.arm_profile().name()); } ``` ```bash -# Automatic SIMD detection +# Portable / distribution build — x86-64-v3 (AVX2) baseline, runs on any +# Haswell-or-later x86_64. Pass the config EXPLICITLY: since 2026-09-16 the +# default is `target-cpu=native`, which tunes the artifact to the BUILD host +# and is not safe to ship (`.cargo/config-native.toml` says so in as many +# words). Runtime `simd_caps()` detection cannot rescue a binary whose +# baseline codegen already emits host-only instructions. +cargo --config .cargo/config-v3.toml build --release + +# Build for THIS machine (dev / benchmarking). Fastest here, portable nowhere. cargo build --release # Cross-compile for Raspberry Pi 4 cargo build --release --target aarch64-unknown-linux-gnu # Maximum performance on AVX-512 server -RUSTFLAGS="-C target-cpu=x86-64-v4" cargo build --release +cargo --config .cargo/config-v4.toml build --release # Run 880 HPC tests cargo test diff --git a/examples/ternlogq_tail_descent_probe.rs b/examples/ternlogq_tail_descent_probe.rs index c5eaabf4..5c12026a 100644 --- a/examples/ternlogq_tail_descent_probe.rs +++ b/examples/ternlogq_tail_descent_probe.rs @@ -305,7 +305,7 @@ fn main() { // ---- the crux: for a tail of t words, which decomposition wins? ---- // t = 6 is the interesting one: 1 ymm + 1 xmm, or 3 xmm, or one padded zmm? println!("== TAIL-ONLY, per remainder length: which split? (ns per call) =="); - println!("{:>5} {:>10} {:>10} {:>10} {}", "t", "P padded", "G greedy", "X all-xmm", "winner"); + println!("{:>5} {:>10} {:>10} {:>10} winner", "t", "P padded", "G greedy", "X all-xmm"); for t in 1..=7usize { let a: Vec = (0..t).map(|i| 0xF0F0_1111u64 ^ i as u64).collect(); let b: Vec = (0..t).map(|i| 0xFF00_2222u64 ^ i as u64).collect(); diff --git a/src/simd_int_ops.rs b/src/simd_int_ops.rs index 390c71d5..24c104b8 100644 --- a/src/simd_int_ops.rs +++ b/src/simd_int_ops.rs @@ -294,6 +294,19 @@ pub fn gemm_u8_i8(a: &[u8], b: &[i8], c: &mut [i32], m: usize, n: usize, k: usiz // Zen 4 silicon that supports VNNI at runtime (the regression codex flagged // on PR #217). Runtime detection keeps the VNNI kernels reachable on the // baseline build, matching the pre-consolidation `simd_caps()` behaviour. + // `allow`, not `expect`, and the distinction is the point: this lint is + // CONFIG-DEPENDENT. The trailing scalar fallback below is `#[cfg(not(any( + // avx512vnni, avxvnni)))]`, so on a build where either IS a compile-time + // feature (e.g. the `target-cpu=native` default on VNNI silicon) the + // fallback is stripped, the second `return` becomes the block's final + // statement, and `needless_return` fires — while on the portable v3 arm the + // fallback is present and BOTH returns are load-bearing. `expect` would + // then fail the v3 build for the lint NOT firing, turning one arm's cleanup + // into the other arm's error. The returns stay symmetric on purpose: two + // parallel runtime-detected arms that read the same way in every config. + // (Surfaced 2026-09-16 by the default flip to `native` — under the old v3 + // default this code path was never linted at its own feature level.) + #[allow(clippy::needless_return)] #[cfg(target_arch = "x86_64")] { if std::is_x86_feature_detected!("avx512vnni") { diff --git a/src/simd_masking_ops.rs b/src/simd_masking_ops.rs index 3ecc13c6..453733be 100644 --- a/src/simd_masking_ops.rs +++ b/src/simd_masking_ops.rs @@ -182,26 +182,10 @@ fn tail_lane_bits_8(n: usize) -> u8 { /// ``` #[inline] pub fn eq_u32_to_mask(values: &[u32], needle: u32, out_words: &mut [u64]) { - let n = values.len(); - let words = mask_words_for(n); - assert!(out_words.len() >= words, "eq_u32_to_mask: out_words.len()={} < required {}", out_words.len(), words); - - // Zero first: makes the "trailing bits are 0" guarantee structural. - for w in out_words.iter_mut() { - *w = 0; - } - let needle_v = crate::simd::U32x16::splat(needle); - let (chunks, tail) = values.as_chunks::<16>(); - for (g, chunk) in chunks.iter().enumerate() { - let bits = crate::simd::U32x16::from_array(*chunk).eq_bitmask(needle_v); - out_words[g / 4] |= (bits as u64) << ((g % 4) * 16); - } - if !tail.is_empty() { - let g = chunks.len(); - let bits = crate::simd::U32x16::from_array(pad_tail(tail)).eq_bitmask(needle_v) & tail_lane_bits(tail.len()); - out_words[g / 4] |= (bits as u64) << ((g % 4) * 16); - } + pack::("eq_u32_to_mask", values, out_words, |lanes, live| { + (crate::simd::U32x16::from_array(lanes).eq_bitmask(needle_v) & live16(live)) as u64 + }); } /// Packs `read_le_u32(bytes, first_offset + i * stride_bytes) == needle` into @@ -358,25 +342,10 @@ pub fn eq_u32_strided_to_mask( /// ``` #[inline] pub fn gt_i32_to_mask(values: &[i32], threshold: i32, out_words: &mut [u64]) { - let n = values.len(); - let words = mask_words_for(n); - assert!(out_words.len() >= words, "gt_i32_to_mask: out_words.len()={} < required {}", out_words.len(), words); - - for w in out_words.iter_mut() { - *w = 0; - } - let threshold_v = crate::simd::I32x16::splat(threshold); - let (chunks, tail) = values.as_chunks::<16>(); - for (g, chunk) in chunks.iter().enumerate() { - let bits = crate::simd::I32x16::from_array(*chunk).gt_bitmask(threshold_v); - out_words[g / 4] |= (bits as u64) << ((g % 4) * 16); - } - if !tail.is_empty() { - let g = chunks.len(); - let bits = crate::simd::I32x16::from_array(pad_tail(tail)).gt_bitmask(threshold_v) & tail_lane_bits(tail.len()); - out_words[g / 4] |= (bits as u64) << ((g % 4) * 16); - } + pack::("gt_i32_to_mask", values, out_words, |lanes, live| { + (crate::simd::I32x16::from_array(lanes).gt_bitmask(threshold_v) & live16(live)) as u64 + }); } /// `dst = a & b`, elementwise over `u64` mask words. @@ -899,23 +868,10 @@ fn clear_mask_tail(out_words: &mut [u64], n: usize) { /// ``` #[inline] pub fn lt_i32_to_mask(values: &[i32], threshold: i32, out_words: &mut [u64]) { - let n = values.len(); - let words = mask_words_for(n); - assert!(out_words.len() >= words, "lt_i32_to_mask: out_words.len()={} < required {}", out_words.len(), words); - for w in out_words.iter_mut() { - *w = 0; - } let t = crate::simd::I32x16::splat(threshold); - let (chunks, tail) = values.as_chunks::<16>(); - for (g, chunk) in chunks.iter().enumerate() { - let bits = t.gt_bitmask(crate::simd::I32x16::from_array(*chunk)); - out_words[g / 4] |= (bits as u64) << ((g % 4) * 16); - } - if !tail.is_empty() { - let g = chunks.len(); - let bits = t.gt_bitmask(crate::simd::I32x16::from_array(pad_tail(tail))) & tail_lane_bits(tail.len()); - out_words[g / 4] |= (bits as u64) << ((g % 4) * 16); - } + pack::("lt_i32_to_mask", values, out_words, |lanes, live| { + (t.gt_bitmask(crate::simd::I32x16::from_array(lanes)) & live16(live)) as u64 + }); } /// Packs `values[i] >= threshold` (signed): the complement of @@ -993,25 +949,11 @@ pub fn le_i32_to_mask(values: &[i32], threshold: i32, out_words: &mut [u64]) { /// ``` #[inline] pub fn ne_i32_to_mask(values: &[i32], needle: i32, out_words: &mut [u64]) { - let n = values.len(); - let words = mask_words_for(n); - assert!(out_words.len() >= words, "ne_i32_to_mask: out_words.len()={} < required {}", out_words.len(), words); - for w in out_words.iter_mut() { - *w = 0; - } let t = crate::simd::I32x16::splat(needle); - let (chunks, tail) = values.as_chunks::<16>(); - for (g, chunk) in chunks.iter().enumerate() { - let v = crate::simd::I32x16::from_array(*chunk); - let bits = t.gt_bitmask(v) | v.gt_bitmask(t); - out_words[g / 4] |= (bits as u64) << ((g % 4) * 16); - } - if !tail.is_empty() { - let g = chunks.len(); - let v = crate::simd::I32x16::from_array(pad_tail(tail)); - let bits = (t.gt_bitmask(v) | v.gt_bitmask(t)) & tail_lane_bits(tail.len()); - out_words[g / 4] |= (bits as u64) << ((g % 4) * 16); - } + pack::("ne_i32_to_mask", values, out_words, |lanes, live| { + let v = crate::simd::I32x16::from_array(lanes); + ((t.gt_bitmask(v) | v.gt_bitmask(t)) & live16(live)) as u64 + }); } /// Packs `values[i] == needle` (signed lanes, exact): the complement of @@ -1130,24 +1072,10 @@ pub fn ne_u32_to_mask(values: &[u32], needle: u32, out_words: &mut [u64]) { /// ``` #[inline] pub fn eq_u8_to_mask(values: &[u8], needle: u8, out_words: &mut [u64]) { - let n = values.len(); - let words = mask_words_for(n); - assert!(out_words.len() >= words, "eq_u8_to_mask: out_words.len()={} < required {}", out_words.len(), words); - - for w in out_words.iter_mut() { - *w = 0; - } - let needle_v = crate::simd::U8x64::splat(needle); - let (chunks, tail) = values.as_chunks::<64>(); - for (g, chunk) in chunks.iter().enumerate() { - out_words[g] = crate::simd::U8x64::from_array(*chunk).cmpeq_mask(needle_v); - } - if !tail.is_empty() { - let g = chunks.len(); - out_words[g] = - crate::simd::U8x64::from_array(pad_tail(tail)).cmpeq_mask(needle_v) & word_range_mask(0, tail.len()); - } + pack::("eq_u8_to_mask", values, out_words, |lanes, live| { + crate::simd::U8x64::from_array(lanes).cmpeq_mask(needle_v) & live64(live) + }); } /// Packs `values[i] > threshold` (**unsigned** comparison — the only @@ -1188,24 +1116,10 @@ pub fn eq_u8_to_mask(values: &[u8], needle: u8, out_words: &mut [u64]) { /// ``` #[inline] pub fn gt_u8_to_mask(values: &[u8], threshold: u8, out_words: &mut [u64]) { - let n = values.len(); - let words = mask_words_for(n); - assert!(out_words.len() >= words, "gt_u8_to_mask: out_words.len()={} < required {}", out_words.len(), words); - - for w in out_words.iter_mut() { - *w = 0; - } - let threshold_v = crate::simd::U8x64::splat(threshold); - let (chunks, tail) = values.as_chunks::<64>(); - for (g, chunk) in chunks.iter().enumerate() { - out_words[g] = crate::simd::U8x64::from_array(*chunk).cmpgt_mask(threshold_v); - } - if !tail.is_empty() { - let g = chunks.len(); - out_words[g] = - crate::simd::U8x64::from_array(pad_tail(tail)).cmpgt_mask(threshold_v) & word_range_mask(0, tail.len()); - } + pack::("gt_u8_to_mask", values, out_words, |lanes, live| { + crate::simd::U8x64::from_array(lanes).cmpgt_mask(threshold_v) & live64(live) + }); } /// Packs `values[i] < threshold` (unsigned): computed directly as @@ -1233,23 +1147,10 @@ pub fn gt_u8_to_mask(values: &[u8], threshold: u8, out_words: &mut [u64]) { /// ``` #[inline] pub fn lt_u8_to_mask(values: &[u8], threshold: u8, out_words: &mut [u64]) { - let n = values.len(); - let words = mask_words_for(n); - assert!(out_words.len() >= words, "lt_u8_to_mask: out_words.len()={} < required {}", out_words.len(), words); - - for w in out_words.iter_mut() { - *w = 0; - } - let t = crate::simd::U8x64::splat(threshold); - let (chunks, tail) = values.as_chunks::<64>(); - for (g, chunk) in chunks.iter().enumerate() { - out_words[g] = t.cmpgt_mask(crate::simd::U8x64::from_array(*chunk)); - } - if !tail.is_empty() { - let g = chunks.len(); - out_words[g] = t.cmpgt_mask(crate::simd::U8x64::from_array(pad_tail(tail))) & word_range_mask(0, tail.len()); - } + pack::("lt_u8_to_mask", values, out_words, |lanes, live| { + t.cmpgt_mask(crate::simd::U8x64::from_array(lanes)) & live64(live) + }); } /// Packs `values[i] >= threshold` (unsigned): the complement of @@ -1389,25 +1290,10 @@ pub fn ne_u8_to_mask(values: &[u8], needle: u8, out_words: &mut [u64]) { /// ``` #[inline] pub fn eq_u64_to_mask(values: &[u64], needle: u64, out_words: &mut [u64]) { - let n = values.len(); - let words = mask_words_for(n); - assert!(out_words.len() >= words, "eq_u64_to_mask: out_words.len()={} < required {}", out_words.len(), words); - - for w in out_words.iter_mut() { - *w = 0; - } - let needle_v = crate::simd::U64x8::splat(needle); - let (chunks, tail) = values.as_chunks::<8>(); - for (g, chunk) in chunks.iter().enumerate() { - let bits = crate::simd::U64x8::from_array(*chunk).cmpeq_mask(needle_v); - out_words[g / 8] |= (bits as u64) << ((g % 8) * 8); - } - if !tail.is_empty() { - let g = chunks.len(); - let bits = crate::simd::U64x8::from_array(pad_tail(tail)).cmpeq_mask(needle_v) & tail_lane_bits_8(tail.len()); - out_words[g / 8] |= (bits as u64) << ((g % 8) * 8); - } + pack::("eq_u64_to_mask", values, out_words, |lanes, live| { + (crate::simd::U64x8::from_array(lanes).cmpeq_mask(needle_v) & live8(live)) as u64 + }); } /// Packs `values[i] > threshold` (**unsigned** — the only ordering `u64` @@ -1438,26 +1324,10 @@ pub fn eq_u64_to_mask(values: &[u64], needle: u64, out_words: &mut [u64]) { /// ``` #[inline] pub fn gt_u64_to_mask(values: &[u64], threshold: u64, out_words: &mut [u64]) { - let n = values.len(); - let words = mask_words_for(n); - assert!(out_words.len() >= words, "gt_u64_to_mask: out_words.len()={} < required {}", out_words.len(), words); - - for w in out_words.iter_mut() { - *w = 0; - } - let threshold_v = crate::simd::U64x8::splat(threshold); - let (chunks, tail) = values.as_chunks::<8>(); - for (g, chunk) in chunks.iter().enumerate() { - let bits = crate::simd::U64x8::from_array(*chunk).cmpgt_mask(threshold_v); - out_words[g / 8] |= (bits as u64) << ((g % 8) * 8); - } - if !tail.is_empty() { - let g = chunks.len(); - let bits = - crate::simd::U64x8::from_array(pad_tail(tail)).cmpgt_mask(threshold_v) & tail_lane_bits_8(tail.len()); - out_words[g / 8] |= (bits as u64) << ((g % 8) * 8); - } + pack::("gt_u64_to_mask", values, out_words, |lanes, live| { + (crate::simd::U64x8::from_array(lanes).cmpgt_mask(threshold_v) & live8(live)) as u64 + }); } /// Packs `values[i] < threshold` (unsigned): computed directly as @@ -1489,25 +1359,10 @@ pub fn gt_u64_to_mask(values: &[u64], threshold: u64, out_words: &mut [u64]) { /// ``` #[inline] pub fn lt_u64_to_mask(values: &[u64], threshold: u64, out_words: &mut [u64]) { - let n = values.len(); - let words = mask_words_for(n); - assert!(out_words.len() >= words, "lt_u64_to_mask: out_words.len()={} < required {}", out_words.len(), words); - - for w in out_words.iter_mut() { - *w = 0; - } - let t = crate::simd::U64x8::splat(threshold); - let (chunks, tail) = values.as_chunks::<8>(); - for (g, chunk) in chunks.iter().enumerate() { - let bits = t.cmpgt_mask(crate::simd::U64x8::from_array(*chunk)); - out_words[g / 8] |= (bits as u64) << ((g % 8) * 8); - } - if !tail.is_empty() { - let g = chunks.len(); - let bits = t.cmpgt_mask(crate::simd::U64x8::from_array(pad_tail(tail))) & tail_lane_bits_8(tail.len()); - out_words[g / 8] |= (bits as u64) << ((g % 8) * 8); - } + pack::("lt_u64_to_mask", values, out_words, |lanes, live| { + (t.cmpgt_mask(crate::simd::U64x8::from_array(lanes)) & live8(live)) as u64 + }); } /// Packs `values[i] >= threshold` (unsigned): the complement of @@ -1948,36 +1803,15 @@ pub fn mask_all(words: &[u64], n_rows: usize) -> bool { /// ``` #[inline] pub fn ternary_match_u32_to_mask(values: &[u32], pattern: u32, care: u32, out_words: &mut [u64]) { - let n = values.len(); - let words = mask_words_for(n); - assert!( - out_words.len() >= words, - "ternary_match_u32_to_mask: out_words.len()={} < required {}", - out_words.len(), - words - ); - for w in out_words.iter_mut() { - *w = 0; - } let p = crate::simd::U32x16::splat(pattern); let c = crate::simd::U32x16::splat(care); let zero = crate::simd::U32x16::splat(0); - let (chunks, tail) = values.as_chunks::<16>(); - for (g, chunk) in chunks.iter().enumerate() { - let v = crate::simd::U32x16::from_array(*chunk); - let bits = v + pack::("ternary_match_u32_to_mask", values, out_words, |lanes, live| { + let bits = crate::simd::U32x16::from_array(lanes) .ternlog::<{ crate::simd::ternlog::XOR_AND }>(p, c) .eq_bitmask(zero); - out_words[g / 4] |= (bits as u64) << ((g % 4) * 16); - } - if !tail.is_empty() { - let g = chunks.len(); - let bits = crate::simd::U32x16::from_array(pad_tail(tail)) - .ternlog::<{ crate::simd::ternlog::XOR_AND }>(p, c) - .eq_bitmask(zero) - & tail_lane_bits(tail.len()); - out_words[g / 4] |= (bits as u64) << ((g % 4) * 16); - } + (bits & live16(live)) as u64 + }); } /// The 64-bit sibling of [`ternary_match_u32_to_mask`]: packs @@ -2001,42 +1835,18 @@ pub fn ternary_match_u32_to_mask(values: &[u32], pattern: u32, care: u32, out_wo /// ``` #[inline] pub fn ternary_match_u64_to_mask(values: &[u64], pattern: u64, care: u64, out_words: &mut [u64]) { - let n = values.len(); - let words = mask_words_for(n); - assert!( - out_words.len() >= words, - "ternary_match_u64_to_mask: out_words.len()={} < required {}", - out_words.len(), - words - ); - for w in out_words.iter_mut() { - *w = 0; - } let p = crate::simd::U64x8::splat(pattern); let c = crate::simd::U64x8::splat(care); - const L: usize = crate::simd::U64x8::LANES; - let (chunks, tail) = values.as_chunks::(); - for (g, chunk) in chunks.iter().enumerate() { - let r = crate::simd::U64x8::from_array(*chunk) - .ternlog::<{ crate::simd::ternlog::XOR_AND }>(p, c) - .to_array(); - let mut bits = 0u64; - for (lane, &x) in r.iter().enumerate() { - bits |= ((x == 0) as u64) << lane; - } - out_words[g / 8] |= bits << ((g % 8) * 8); - } - if !tail.is_empty() { - let g = chunks.len(); - let r = crate::simd::U64x8::from_array(pad_tail(tail)) + pack::("ternary_match_u64_to_mask", values, out_words, |lanes, live| { + let r = crate::simd::U64x8::from_array(lanes) .ternlog::<{ crate::simd::ternlog::XOR_AND }>(p, c) .to_array(); let mut bits = 0u64; - for (lane, &x) in r.iter().take(tail.len()).enumerate() { + for (lane, &x) in r.iter().take(live).enumerate() { bits |= ((x == 0) as u64) << lane; } - out_words[g / 8] |= bits << ((g % 8) * 8); - } + bits + }); } /// Care-masked match of a **12-byte little-endian register** found at @@ -2237,6 +2047,68 @@ fn live16(live: usize) -> u16 { } } +/// `live`-lane validity mask for an 8-lane group: all ones for a full group, +/// [`tail_lane_bits_8`] for a padded one. +#[inline(always)] +fn live8(live: usize) -> u8 { + if live == 8 { + u8::MAX + } else { + tail_lane_bits_8(live) + } +} + +/// `live`-lane validity mask for a 64-lane group (one `u8x64` register is one +/// whole mask word): all ones for a full group, the low `live` bits otherwise. +#[inline(always)] +fn live64(live: usize) -> u64 { + if live == 64 { + u64::MAX + } else { + word_range_mask(0, live) + } +} + +/// The shared full-scan packer under every contiguous `*_to_mask` predicate — +/// the ungated sibling of [`pack_under`]. +/// +/// `group_bits(lanes, live)` packs one register's worth of predicate results: +/// `lanes` is the full `[T; L]` (zero-padded when `live < L`, i.e. only for +/// the final group) and the closure masks its answer down to the low `live` +/// bits. The body iterates `as_chunks::()` — fixed-size loads, no index +/// arithmetic — and the remainder is the ONE spelling of the predicate tail: +/// zero-padded into a register, run through the same closure as the body, +/// masked to `live`. This is where the 12 hand-rolled `if !tail.is_empty()` +/// branches that preceded it were folded (2026-09-16); the mask-algebra tails +/// (`mask_and` and friends) are a different kind and are NOT served here. +#[inline(always)] +fn pack( + name: &str, values: &[T], out_words: &mut [u64], group_bits: impl Fn([T; L], usize) -> u64, +) { + const { assert!(64 % L == 0, "a mask word must hold whole lane groups") } + let k = 64 / L; + let n = values.len(); + let words = mask_words_for(n); + assert!(out_words.len() >= words, "{name}: out_words.len()={} < required {words}", out_words.len()); + // Zero first: makes the "trailing bits are 0" guarantee structural. + for w in out_words.iter_mut() { + *w = 0; + } + let (chunks, tail) = values.as_chunks::(); + for (g, chunk) in chunks.iter().enumerate() { + out_words[g / k] |= group_bits(*chunk, L) << ((g % k) * L); + } + if !tail.is_empty() { + let g = chunks.len(); + let live = tail.len(); + let bits = group_bits(pad_tail(tail), live); + // The closure contract (bits above `live` are zero) is what keeps a + // padding lane from ever contributing a match. + debug_assert!(live == 64 || bits >> live == 0, "{name}: tail bits above live={live} must be zero"); + out_words[g / k] |= bits << ((g % k) * L); + } +} + /// Packs `values[i] > threshold` (signed) AND `under`: bit `i` is set iff /// both hold, and words where `under` has no survivor are written zero /// **without evaluating the compare** — the survivor-word skip whose cost diff --git a/src/simd_nightly/i8_types.rs b/src/simd_nightly/i8_types.rs index acd2e9b2..6bd00e74 100644 --- a/src/simd_nightly/i8_types.rs +++ b/src/simd_nightly/i8_types.rs @@ -37,6 +37,12 @@ impl I8x64 { Self(i8x64::splat(v)) } + /// All lanes zero. + #[inline(always)] + pub fn zero() -> Self { + Self::splat(0) + } + /// Load from a slice of at least 64 elements (panics otherwise). #[inline(always)] pub fn from_slice(s: &[i8]) -> Self { @@ -129,6 +135,18 @@ impl I8x64 { Self(self.0.simd_max(other.0)) } + /// Lane-wise wrapping add — mirrors `simd_avx512::I8x64::add` (`_mm512_add_epi8`). + #[inline(always)] + pub fn add(self, other: Self) -> Self { + Self(self.0 + other.0) + } + + /// Lane-wise wrapping subtract — mirrors `simd_avx512::I8x64::sub` (`_mm512_sub_epi8`). + #[inline(always)] + pub fn sub(self, other: Self) -> Self { + Self(self.0 - other.0) + } + /// Saturating absolute value: `|i8::MIN|` is `i8::MAX` (127), never the /// wrapped `i8::MIN` — the crate's `saturating_abs` contract (see the /// VPABSB correction in `vertical-simd-consumer-contract.md`). @@ -178,6 +196,12 @@ impl I8x64 { pub fn cmpgt_mask(self, other: Self) -> u64 { self.0.simd_gt(other.0).to_bitmask() } + + /// Bitmask of `self > other`, one bit per lane — mirrors `simd_avx512::I8x64::cmp_gt`. + #[inline(always)] + pub fn cmp_gt(self, other: Self) -> u64 { + self.cmpgt_mask(other) + } } impl PartialEq for I8x64 { @@ -216,6 +240,12 @@ impl I8x32 { Self(i8x32::splat(v)) } + /// All lanes zero. + #[inline(always)] + pub fn zero() -> Self { + Self::splat(0) + } + /// Load from a slice of at least 32 elements (panics otherwise). #[inline(always)] pub fn from_slice(s: &[i8]) -> Self { @@ -308,6 +338,18 @@ impl I8x32 { Self(self.0.simd_max(other.0)) } + /// Lane-wise wrapping add — mirrors `simd_avx512::I8x32::add` (`_mm512_add_epi8`). + #[inline(always)] + pub fn add(self, other: Self) -> Self { + Self(self.0 + other.0) + } + + /// Lane-wise wrapping subtract — mirrors `simd_avx512::I8x32::sub` (`_mm512_sub_epi8`). + #[inline(always)] + pub fn sub(self, other: Self) -> Self { + Self(self.0 - other.0) + } + /// Saturating absolute value: `|i8::MIN|` is `i8::MAX` (127), never the /// wrapped `i8::MIN` — the crate's `saturating_abs` contract (see the /// VPABSB correction in `vertical-simd-consumer-contract.md`). @@ -357,6 +399,12 @@ impl I8x32 { pub fn cmpgt_mask(self, other: Self) -> u32 { self.0.simd_gt(other.0).to_bitmask() as u32 } + + /// Bitmask of `self > other`, one bit per lane — mirrors `simd_avx512::I8x32::cmp_gt`. + #[inline(always)] + pub fn cmp_gt(self, other: Self) -> u32 { + self.cmpgt_mask(other) + } } impl PartialEq for I8x32 { diff --git a/src/simd_nightly/i_word_types.rs b/src/simd_nightly/i_word_types.rs index c42f3b3a..3afd2878 100644 --- a/src/simd_nightly/i_word_types.rs +++ b/src/simd_nightly/i_word_types.rs @@ -24,6 +24,12 @@ impl I16x16 { Self(i16x16::splat(v)) } + /// All lanes zero. + #[inline(always)] + pub fn zero() -> Self { + Self::splat(0) + } + #[inline(always)] pub fn from_array(arr: [i16; 16]) -> Self { Self(i16x16::from_array(arr)) @@ -76,6 +82,30 @@ impl I16x16 { Self(self.0.simd_max(other.0)) } + /// Lane-wise signed minimum — mirrors `simd_avx512::I16x16::min` (`_mm512_min_epi16`). + #[inline(always)] + pub fn min(self, other: Self) -> Self { + Self(self.0.simd_min(other.0)) + } + + /// Lane-wise signed maximum — mirrors `simd_avx512::I16x16::max` (`_mm512_max_epi16`). + #[inline(always)] + pub fn max(self, other: Self) -> Self { + Self(self.0.simd_max(other.0)) + } + + /// Lane-wise wrapping add — mirrors `simd_avx512::I16x16::add` (`_mm512_add_epi16`). + #[inline(always)] + pub fn add(self, other: Self) -> Self { + Self(self.0 + other.0) + } + + /// Lane-wise wrapping subtract — mirrors `simd_avx512::I16x16::sub` (`_mm512_sub_epi16`). + #[inline(always)] + pub fn sub(self, other: Self) -> Self { + Self(self.0 - other.0) + } + // ── Saturating arithmetic ───────────────────────────────────── #[inline(always)] @@ -102,6 +132,12 @@ impl I16x16 { pub fn cmpgt_mask(self, other: Self) -> u16 { self.0.simd_gt(other.0).to_bitmask() as u16 } + + /// Bitmask of `self > other`, one bit per lane — mirrors `simd_avx512::I16x16::cmp_gt`. + #[inline(always)] + pub fn cmp_gt(self, other: Self) -> u16 { + self.cmpgt_mask(other) + } } impl PartialEq for I16x16 { @@ -136,6 +172,12 @@ impl I16x32 { Self(i16x32::splat(v)) } + /// All lanes zero. + #[inline(always)] + pub fn zero() -> Self { + Self::splat(0) + } + #[inline(always)] pub fn from_array(arr: [i16; 32]) -> Self { Self(i16x32::from_array(arr)) @@ -188,6 +230,30 @@ impl I16x32 { Self(self.0.simd_max(other.0)) } + /// Lane-wise signed minimum — mirrors `simd_avx512::I16x32::min` (`_mm512_min_epi16`). + #[inline(always)] + pub fn min(self, other: Self) -> Self { + Self(self.0.simd_min(other.0)) + } + + /// Lane-wise signed maximum — mirrors `simd_avx512::I16x32::max` (`_mm512_max_epi16`). + #[inline(always)] + pub fn max(self, other: Self) -> Self { + Self(self.0.simd_max(other.0)) + } + + /// Lane-wise wrapping add — mirrors `simd_avx512::I16x32::add` (`_mm512_add_epi16`). + #[inline(always)] + pub fn add(self, other: Self) -> Self { + Self(self.0 + other.0) + } + + /// Lane-wise wrapping subtract — mirrors `simd_avx512::I16x32::sub` (`_mm512_sub_epi16`). + #[inline(always)] + pub fn sub(self, other: Self) -> Self { + Self(self.0 - other.0) + } + // ── Saturating arithmetic ───────────────────────────────────── #[inline(always)] @@ -214,6 +280,12 @@ impl I16x32 { pub fn cmpgt_mask(self, other: Self) -> u32 { self.0.simd_gt(other.0).to_bitmask() as u32 } + + /// Bitmask of `self > other`, one bit per lane — mirrors `simd_avx512::I16x32::cmp_gt`. + #[inline(always)] + pub fn cmp_gt(self, other: Self) -> u32 { + self.cmpgt_mask(other) + } } impl PartialEq for I16x32 {