From 5287b6ab9aa2b6dd7aacae8754549a5ddba72e05 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 29 Jul 2026 08:44:32 +0000 Subject: [PATCH 1/2] simd: add the U32x8 shuffle + rotate surface for the BLAKE3 port BLAKE3's pure-Rust AVX2 backend (rust_avx2.rs) is hash_many at DEGREE = 8, so it is U32x8-shaped rather than U32x16-shaped. Porting it off raw core::arch needs exactly six operations beyond what avx2_int_type! generates: rotate_left, the 32- and 64-bit unpacks, and the two 128-bit half concatenations its transpose network uses. Every body is a plain scalar index loop, deliberately. The codegen oracle measured this exact shape: a fixed two-source permutation written as an index loop compiles to a real packed shuffle (vpermd + vpblendd), and a transpose composed from such primitives emits vpunpcklqdq / vpermq / vinserti128 -- the same instruction family the hand-written intrinsics use. No unsafe, no core::arch, no intrinsic override earned. Semantics are x86's, INCLUDING the per-128-bit-lane split, on every backend. That is not an x86 leak: BLAKE3's transpose is defined in terms of it, so a backend that "helpfully" used whole-vector interleave would compute a different permutation and produce wrong hashes with no compile error. Added on the avx2 arm (where U32x8 is avx2_int_type!-generated) and mirrored on the scalar arm; avx512 re-exports the avx2 type and gets it for free. Only two backends define U32x8 -- neon/wasm/nightly do not, and do not need to, since rust_avx2.rs is x86-gated and blake3 has separate backends for those. Only the two permute2x128 immediates BLAKE3 uses (0x20, 0x31) are exposed, as named methods rather than a generic const-IMM permute: a full 2x128 permute is a much larger surface whose remaining immediates would have no caller and no parity test. Two tests, and the first is the load-bearing one: - u32x8_shuffles_match_x86_intrinsics asserts each method against the REAL intrinsic it claims to reproduce, and the four rotates against upstream's exact `srli | slli` form rather than a reformulation of our own. Gated on target_feature = "avx2". - u32x8_shuffle_surface_is_lane_exact pins the same contract lane-by-lane with no intrinsics, so scalar/avx512/any future arm cannot diverge on a machine where the oracle is unavailable. Control-tested: rewriting interleave_lo_u32 as the "obvious" whole-vector form -- the precise mistake the per-128-lane semantics exist to prevent -- fails BOTH tests. They discriminate; they are not vacuous assertions. Recorded for the port that follows: upstream's rot16/rot8 use the shift-or form deliberately, citing LLVM bug 44379 and a measured preference on recent x86, whereas a scalar rotate_left may fold to vpshufb for byte-granular amounts. That is a codegen difference, not a semantic one -- values are identical either way -- but it is why the port needs a throughput comparison and not only a parity test. --- src/simd.rs | 128 +++++++++++++++++++++++++++++++++++++++++++++ src/simd_avx2.rs | 98 ++++++++++++++++++++++++++++++++++ src/simd_scalar.rs | 68 ++++++++++++++++++++++++ 3 files changed, 294 insertions(+) diff --git a/src/simd.rs b/src/simd.rs index ce869ba1..5e78a6fe 100644 --- a/src/simd.rs +++ b/src/simd.rs @@ -743,6 +743,134 @@ mod tests { } } + /// `U32x8`'s shuffle + rotate surface, checked against the REAL x86 + /// intrinsics it claims to reproduce. + /// + /// This is the load-bearing test for the BLAKE3 `rust_avx2.rs` port. Those + /// six methods have plain index-loop bodies, so nothing about them is + /// self-evidently equal to `_mm256_unpacklo_epi32` and friends — and the + /// per-128-bit-lane semantics are exactly the kind of thing an "obvious" + /// whole-vector implementation gets wrong while still looking reasonable. + /// A transpose built on a subtly-wrong interleave produces wrong hashes, + /// not a compile error. + /// + /// Gated on `target_feature = "avx2"` because it calls the intrinsics + /// directly as the oracle. The methods themselves are backend-agnostic; + /// `u32x8_shuffle_surface_is_lane_exact` below covers them everywhere. + #[cfg(all(target_arch = "x86_64", target_feature = "avx2"))] + #[test] + fn u32x8_shuffles_match_x86_intrinsics() { + use super::U32x8; + use core::arch::x86_64::*; + + // Distinct values in every lane, so any misplaced lane is visible. + let a_arr: [u32; 8] = [ + 0x1000_0000, 0x1100_0001, 0x1200_0002, 0x1300_0003, 0x1400_0004, 0x1500_0005, 0x1600_0006, 0x1700_0007, + ]; + let b_arr: [u32; 8] = [ + 0x2000_0000, 0x2100_0001, 0x2200_0002, 0x2300_0003, 0x2400_0004, 0x2500_0005, 0x2600_0006, 0x2700_0007, + ]; + let (a, b) = (U32x8::from_array(a_arr), U32x8::from_array(b_arr)); + + // SAFETY: guarded by `target_feature = "avx2"` on this test. + unsafe { + let va = _mm256_loadu_si256(a_arr.as_ptr() as *const __m256i); + let vb = _mm256_loadu_si256(b_arr.as_ptr() as *const __m256i); + let read = |v: __m256i| -> [u32; 8] { + let mut out = [0u32; 8]; + _mm256_storeu_si256(out.as_mut_ptr() as *mut __m256i, v); + out + }; + + assert_eq!( + a.interleave_lo_u32(b).to_array(), + read(_mm256_unpacklo_epi32(va, vb)), + "interleave_lo_u32 != _mm256_unpacklo_epi32" + ); + assert_eq!( + a.interleave_hi_u32(b).to_array(), + read(_mm256_unpackhi_epi32(va, vb)), + "interleave_hi_u32 != _mm256_unpackhi_epi32" + ); + assert_eq!( + a.interleave_lo_u64(b).to_array(), + read(_mm256_unpacklo_epi64(va, vb)), + "interleave_lo_u64 != _mm256_unpacklo_epi64" + ); + assert_eq!( + a.interleave_hi_u64(b).to_array(), + read(_mm256_unpackhi_epi64(va, vb)), + "interleave_hi_u64 != _mm256_unpackhi_epi64" + ); + assert_eq!( + a.concat_lo_halves(b).to_array(), + read(_mm256_permute2x128_si256(va, vb, 0x20)), + "concat_lo_halves != _mm256_permute2x128_si256(_, _, 0x20)" + ); + assert_eq!( + a.concat_hi_halves(b).to_array(), + read(_mm256_permute2x128_si256(va, vb, 0x31)), + "concat_hi_halves != _mm256_permute2x128_si256(_, _, 0x31)" + ); + + // BLAKE3 rotates RIGHT by 16/12/8/7, expressed as rotl(32 - n). + // Upstream implements each as `srli | slli`; assert against that + // exact form, not against a reformulation of our own. Written out + // four times rather than looped because the shift intrinsics take + // const immediates -- which also makes this mirror upstream's four + // separate `rot16` / `rot12` / `rot8` / `rot7` functions one-to-one. + assert_eq!( + a.rotate_left(16).to_array(), + read(_mm256_or_si256(_mm256_srli_epi32(va, 16), _mm256_slli_epi32(va, 16))), + "rotate_left(16) != upstream rot16" + ); + assert_eq!( + a.rotate_left(20).to_array(), + read(_mm256_or_si256(_mm256_srli_epi32(va, 12), _mm256_slli_epi32(va, 20))), + "rotate_left(20) != upstream rot12" + ); + assert_eq!( + a.rotate_left(24).to_array(), + read(_mm256_or_si256(_mm256_srli_epi32(va, 8), _mm256_slli_epi32(va, 24))), + "rotate_left(24) != upstream rot8" + ); + assert_eq!( + a.rotate_left(25).to_array(), + read(_mm256_or_si256(_mm256_srli_epi32(va, 7), _mm256_slli_epi32(va, 25))), + "rotate_left(25) != upstream rot7" + ); + } + } + + /// The same six methods, asserted lane-by-lane against their documented + /// index formulas — on EVERY backend, with no intrinsics involved. + /// + /// The intrinsic test above can only run where AVX2 exists. This one + /// pins the contract on scalar / avx512 / any future arm, so a backend + /// cannot quietly diverge on a machine where the oracle is unavailable. + #[test] + fn u32x8_shuffle_surface_is_lane_exact() { + use super::U32x8; + + let a: [u32; 8] = [10, 11, 12, 13, 14, 15, 16, 17]; + let b: [u32; 8] = [20, 21, 22, 23, 24, 25, 26, 27]; + let (va, vb) = (U32x8::from_array(a), U32x8::from_array(b)); + + assert_eq!(va.interleave_lo_u32(vb).to_array(), [10, 20, 11, 21, 14, 24, 15, 25]); + assert_eq!(va.interleave_hi_u32(vb).to_array(), [12, 22, 13, 23, 16, 26, 17, 27]); + assert_eq!(va.interleave_lo_u64(vb).to_array(), [10, 11, 20, 21, 14, 15, 24, 25]); + assert_eq!(va.interleave_hi_u64(vb).to_array(), [12, 13, 22, 23, 16, 17, 26, 27]); + assert_eq!(va.concat_lo_halves(vb).to_array(), [10, 11, 12, 13, 20, 21, 22, 23]); + assert_eq!(va.concat_hi_halves(vb).to_array(), [14, 15, 16, 17, 24, 25, 26, 27]); + + for n in [0u32, 1, 7, 16, 20, 24, 25, 31] { + let got = va.rotate_left(n).to_array(); + for i in 0..8 { + assert_eq!(got[i], a[i].rotate_left(n), "lane {i} rotate_left({n})"); + } + } + } + #[test] fn f32x16_add_sub_mul_div() { let a = F32x16::splat(6.0); diff --git a/src/simd_avx2.rs b/src/simd_avx2.rs index 1b00235e..8a5067d3 100644 --- a/src/simd_avx2.rs +++ b/src/simd_avx2.rs @@ -1789,6 +1789,104 @@ avx2_int_type!(U64x4, u64, 4, 0u64); avx2_int_type!(I32x8, i32, 8, 0i32); avx2_int_type!(I64x4, i64, 4, 0i64); +// ── U32x8 shuffle + rotate surface — the BLAKE3 `rust_avx2.rs` port ───────── +// +// BLAKE3's pure-Rust AVX2 backend is `hash_many` at DEGREE = 8, so it is +// `U32x8`-shaped, not `U32x16`-shaped. It needs exactly six operations beyond +// what `avx2_int_type!` generates: a lane rotate, the 32- and 64-bit unpacks, +// and the two 128-bit half-concatenations its transpose network uses. +// +// **Every body here is a plain scalar index loop, deliberately.** The codegen +// oracle (`.claude/knowledge/simd-codegen-oracle/`) measured this exact shape: +// a fixed two-source permutation written as an index loop compiles to a real +// packed shuffle (`vpermd` + `vpblendd`), and a transpose composed from such +// primitives emits `vpunpcklqdq` / `vpermq` / `vinserti128` — the same +// instruction family the hand-written intrinsics use. No `unsafe`, no +// `core::arch`, no intrinsic override earned. See +// `.claude/knowledge/blake3-on-ndarray-simd.md`. +// +// Semantics match x86 EXACTLY, including its per-128-bit-lane behaviour — +// the transpose network depends on it, so an "obvious" whole-vector +// interleave would silently produce a different permutation. Each method +// names the intrinsic it reproduces, and `simd.rs` carries parity tests that +// assert the correspondence against the real intrinsic. +impl U32x8 { + /// Lane-wise `u32::rotate_left(n)`. + /// + /// BLAKE3 specifies RIGHT rotations; express `rotr(n)` as + /// `rotate_left(32 - n)` — exact, since rotation is modular. + /// + /// Note for porters: upstream's `rot16`/`rot8` deliberately use the + /// `srli | slli` shift-or form rather than `_mm256_shuffle_epi8`, citing + /// LLVM bug 44379 and a measured preference on recent x86. A scalar + /// `rotate_left` loop may instead fold to `vpshufb` for byte-granular + /// amounts. That is a codegen difference from upstream, not a semantic + /// one — the values are identical either way — but it is the reason the + /// port needs a throughput comparison and not only a parity test. + #[inline(always)] + pub fn rotate_left(self, n: u32) -> Self { + let mut out = [0u32; 8]; + for i in 0..8 { + out[i] = self.0[i].rotate_left(n); + } + Self(out) + } + + /// `_mm256_unpacklo_epi32` — interleave the LOW two `u32` of each + /// 128-bit half: `[a0,b0,a1,b1, a4,b4,a5,b5]`. + #[inline(always)] + pub fn interleave_lo_u32(self, other: Self) -> Self { + let (a, b) = (self.0, other.0); + Self([a[0], b[0], a[1], b[1], a[4], b[4], a[5], b[5]]) + } + + /// `_mm256_unpackhi_epi32` — interleave the HIGH two `u32` of each + /// 128-bit half: `[a2,b2,a3,b3, a6,b6,a7,b7]`. + #[inline(always)] + pub fn interleave_hi_u32(self, other: Self) -> Self { + let (a, b) = (self.0, other.0); + Self([a[2], b[2], a[3], b[3], a[6], b[6], a[7], b[7]]) + } + + /// `_mm256_unpacklo_epi64` — interleave the LOW `u64` of each 128-bit + /// half: `[a0,a1,b0,b1, a4,a5,b4,b5]`. + #[inline(always)] + pub fn interleave_lo_u64(self, other: Self) -> Self { + let (a, b) = (self.0, other.0); + Self([a[0], a[1], b[0], b[1], a[4], a[5], b[4], b[5]]) + } + + /// `_mm256_unpackhi_epi64` — interleave the HIGH `u64` of each 128-bit + /// half: `[a2,a3,b2,b3, a6,a7,b6,b7]`. + #[inline(always)] + pub fn interleave_hi_u64(self, other: Self) -> Self { + let (a, b) = (self.0, other.0); + Self([a[2], a[3], b[2], b[3], a[6], a[7], b[6], b[7]]) + } + + /// `_mm256_permute2x128_si256(a, b, 0x20)` — the two LOW 128-bit halves + /// concatenated: `[a0,a1,a2,a3, b0,b1,b2,b3]`. + /// + /// Only the two immediates BLAKE3's transpose uses (`0x20`, `0x31`) are + /// exposed, as named methods rather than a generic `const IMM` permute. + /// A full 2x128 permute is a much larger surface whose remaining + /// immediates have no caller and no parity test; these two are each one + /// line and fully verified. + #[inline(always)] + pub fn concat_lo_halves(self, other: Self) -> Self { + let (a, b) = (self.0, other.0); + Self([a[0], a[1], a[2], a[3], b[0], b[1], b[2], b[3]]) + } + + /// `_mm256_permute2x128_si256(a, b, 0x31)` — the two HIGH 128-bit halves + /// concatenated: `[a4,a5,a6,a7, b4,b5,b6,b7]`. + #[inline(always)] + pub fn concat_hi_halves(self, other: Self) -> Self { + let (a, b) = (self.0, other.0); + Self([a[4], a[5], a[6], a[7], b[4], b[5], b[6], b[7]]) + } +} + // ── W1a SIMD primitives — AVX2 polyfill backend ────────────────────────────── // // The AVX2 backend uses scalar-storage polyfills for the integer types. diff --git a/src/simd_scalar.rs b/src/simd_scalar.rs index 952825ce..1e615e5d 100644 --- a/src/simd_scalar.rs +++ b/src/simd_scalar.rs @@ -542,6 +542,74 @@ impl_int_type!(U64x4, u64, 4, 0u64); impl_int_type!(I32x8, i32, 8, 0i32); impl_int_type!(I64x4, i64, 4, 0i64); +// ── U32x8 shuffle + rotate surface — scalar mirror of `simd_avx2.rs` ──────── +// +// Same six methods, same semantics, same doc contract. The AVX2 arm's bodies +// are ALSO plain index loops (see the long comment there and +// `.claude/knowledge/blake3-on-ndarray-simd.md`), so this is not a "fallback" +// that behaves differently — the two arms are the same source shape, and the +// parity tests in `simd.rs` bind them to the real x86 intrinsics on the one +// backend where those exist. +// +// Semantics are x86's, INCLUDING the per-128-bit-lane split, on every +// backend. That is not an x86 leak: BLAKE3's transpose network is defined in +// terms of it, so a backend that "helpfully" used whole-vector interleave +// would compute a different permutation and produce wrong hashes. +impl U32x8 { + /// Lane-wise `u32::rotate_left(n)`. See the AVX2 arm for the + /// `rotr(n) == rotate_left(32 - n)` note. + #[inline(always)] + pub fn rotate_left(self, n: u32) -> Self { + let mut out = [0u32; 8]; + for i in 0..8 { + out[i] = self.0[i].rotate_left(n); + } + Self(out) + } + + /// `_mm256_unpacklo_epi32`: `[a0,b0,a1,b1, a4,b4,a5,b5]`. + #[inline(always)] + pub fn interleave_lo_u32(self, other: Self) -> Self { + let (a, b) = (self.0, other.0); + Self([a[0], b[0], a[1], b[1], a[4], b[4], a[5], b[5]]) + } + + /// `_mm256_unpackhi_epi32`: `[a2,b2,a3,b3, a6,b6,a7,b7]`. + #[inline(always)] + pub fn interleave_hi_u32(self, other: Self) -> Self { + let (a, b) = (self.0, other.0); + Self([a[2], b[2], a[3], b[3], a[6], b[6], a[7], b[7]]) + } + + /// `_mm256_unpacklo_epi64`: `[a0,a1,b0,b1, a4,a5,b4,b5]`. + #[inline(always)] + pub fn interleave_lo_u64(self, other: Self) -> Self { + let (a, b) = (self.0, other.0); + Self([a[0], a[1], b[0], b[1], a[4], a[5], b[4], b[5]]) + } + + /// `_mm256_unpackhi_epi64`: `[a2,a3,b2,b3, a6,a7,b6,b7]`. + #[inline(always)] + pub fn interleave_hi_u64(self, other: Self) -> Self { + let (a, b) = (self.0, other.0); + Self([a[2], a[3], b[2], b[3], a[6], a[7], b[6], b[7]]) + } + + /// `_mm256_permute2x128_si256(a, b, 0x20)`: `[a0..a3, b0..b3]`. + #[inline(always)] + pub fn concat_lo_halves(self, other: Self) -> Self { + let (a, b) = (self.0, other.0); + Self([a[0], a[1], a[2], a[3], b[0], b[1], b[2], b[3]]) + } + + /// `_mm256_permute2x128_si256(a, b, 0x31)`: `[a4..a7, b4..b7]`. + #[inline(always)] + pub fn concat_hi_halves(self, other: Self) -> Self { + let (a, b) = (self.0, other.0); + Self([a[4], a[5], a[6], a[7], b[4], b[5], b[6], b[7]]) + } +} + // I8x64 / I8x32 / I16x32 / I16x16 — AVX-512BW-style methods (scalar shape) impl I8x64 { #[inline(always)] From 5d4d386dbd4feba49b51d3233ad0f003b4d278a6 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 29 Jul 2026 08:58:14 +0000 Subject: [PATCH 2/2] simd: BLAKE3 shuffle surface on U32x16, all six backends Supersedes this branch's first commit, which put the surface on U32x8. That was wrong and the operator called it: two __m256i fit in one U32x16. BLAKE3's AVX2 backend is DEGREE 8 over __m256i, and I treated that as requiring a matching 8-wide type. It does not. Every operation in hash_many is either lane-wise (add / xor / rotate) or confined WITHIN a 128- or 256-bit lane -- the unpacks are per-128, permute2x128 is per-256 -- so two 8-lane groups packed into one U32x16 never interact. The algorithm runs on both at once as DEGREE 16 with no cross-talk. This is also what the 2026-07-28 ruling already said: a half-width type standing in for the lane the substrate actually uses is an absolute no-go. Practical consequences of being on the right type: - rotate_left already exists on U32x16 on every backend, so only the six shuffles were missing rather than seven ops. - U32x8 is absent from neon, wasm and nightly; U32x16 is on all six. The surface is now uniform, which also answers codex's P1 on #267 (the nightly arm re-exports its own U32x8 without the added methods, so the previous commit's unconditional test could not compile under nightly-simd). - Every backend's U32x16 has to_array/from_array, so ONE identical body serves all six -- no per-backend storage access. Semantics are x86's applied to each 256-bit half independently: interleave_{lo,hi}_u{32,64} reproduce _mm256_unpack{lo,hi}_epi{32,64} within each 128-bit quad, concat_{lo,hi}_halves reproduce _mm256_permute2x128_si256(_, _, 0x20 / 0x31) within each 256-bit half. The per-lane structure is load-bearing: BLAKE3's transpose is defined in terms of it, so a whole-vector interleave would compute a different permutation and produce wrong hashes with no compile error. Bodies are plain index loops, per the oracle's measurement that fixed two-source permutations written that way compile to real packed shuffles. No unsafe, no core::arch, no intrinsic override earned. Two tests. u32x16_blake3_shuffles_match_x86_intrinsics_per_half asserts each half of the U32x16 result against the 256-bit intrinsic applied to that half's inputs, and the four rotates against upstream's exact `srli | slli` form. u32x16_blake3_shuffles_are_lane_exact pins the same permutation with no intrinsics so scalar/avx512/neon/wasm/nightly are held to it where the x86 oracle cannot run. Control-tested: rewriting interleave_lo_u32 as a whole-vector interleave -- precisely the mistake the quad semantics prevent -- fails both tests. 31 simd tests pass; clippy clean; fmt clean. --- src/simd.rs | 103 +++++++++++++++++++++++++ src/simd_avx2.rs | 127 ++++++++++++++++++++++++++++++ src/simd_avx512.rs | 127 ++++++++++++++++++++++++++++++ src/simd_neon.rs | 128 +++++++++++++++++++++++++++++++ src/simd_nightly/u_word_types.rs | 127 ++++++++++++++++++++++++++++++ src/simd_scalar.rs | 127 ++++++++++++++++++++++++++++++ src/simd_wasm.rs | 127 ++++++++++++++++++++++++++++++ 7 files changed, 866 insertions(+) diff --git a/src/simd.rs b/src/simd.rs index 5e78a6fe..04a2735c 100644 --- a/src/simd.rs +++ b/src/simd.rs @@ -743,6 +743,109 @@ mod tests { } } + /// The BLAKE3 shuffle surface on `U32x16`, checked against the REAL x86 + /// intrinsics it reproduces — applied to each 256-bit half. + /// + /// `U32x16` holds two `__m256i` worth of lanes, and BLAKE3's `hash_many` + /// is either lane-wise or confined within a 128-/256-bit lane, so the two + /// 8-lane groups run independently in one vector at DEGREE 16. This test + /// proves that equivalence rather than assuming it: each half of the + /// `U32x16` result must equal the corresponding 256-bit intrinsic applied + /// to that half's inputs. + /// + /// Load-bearing because the bodies are index loops. Nothing about them is + /// self-evidently `_mm256_unpacklo_epi32`, and a subtly-wrong interleave + /// yields wrong hashes, not a compile error. + #[cfg(all(target_arch = "x86_64", target_feature = "avx2"))] + #[test] + fn u32x16_blake3_shuffles_match_x86_intrinsics_per_half() { + use core::arch::x86_64::*; + + // Distinct value per lane, so any misplaced lane is visible. + let a_arr: [u32; 16] = core::array::from_fn(|i| 0x1000_0000 | (i as u32) << 8 | i as u32); + let b_arr: [u32; 16] = core::array::from_fn(|i| 0x2000_0000 | (i as u32) << 8 | i as u32); + let (a, b) = (U32x16::from_array(a_arr), U32x16::from_array(b_arr)); + + // SAFETY: guarded by `target_feature = "avx2"` on this test. + unsafe { + let load = |src: &[u32; 16], half: usize| _mm256_loadu_si256(src.as_ptr().add(half * 8) as *const __m256i); + let read = |v: __m256i| -> [u32; 8] { + let mut out = [0u32; 8]; + _mm256_storeu_si256(out.as_mut_ptr() as *mut __m256i, v); + out + }; + // Assert `got`'s two halves against the 256-bit intrinsic applied + // to each half's own inputs. + let check = |got: U32x16, want: fn(__m256i, __m256i) -> __m256i, name: &str| { + let g = got.to_array(); + for half in 0..2 { + let expect = read(want(load(&a_arr, half), load(&b_arr, half))); + assert_eq!(&g[half * 8..half * 8 + 8], &expect[..], "{name}, half {half}"); + } + }; + + check(a.interleave_lo_u32(b), |x, y| _mm256_unpacklo_epi32(x, y), "interleave_lo_u32"); + check(a.interleave_hi_u32(b), |x, y| _mm256_unpackhi_epi32(x, y), "interleave_hi_u32"); + check(a.interleave_lo_u64(b), |x, y| _mm256_unpacklo_epi64(x, y), "interleave_lo_u64"); + check(a.interleave_hi_u64(b), |x, y| _mm256_unpackhi_epi64(x, y), "interleave_hi_u64"); + check(a.concat_lo_halves(b), |x, y| _mm256_permute2x128_si256(x, y, 0x20), "concat_lo_halves"); + check(a.concat_hi_halves(b), |x, y| _mm256_permute2x128_si256(x, y, 0x31), "concat_hi_halves"); + + // BLAKE3 rotates RIGHT by 16/12/8/7 -> rotl(32 - n). Assert against + // upstream's exact `srli | slli` form, not a reformulation of ours. + // Unrolled because the shift intrinsics take const immediates -- + // which also mirrors upstream's four separate rot fns one-to-one. + let rot = |got: U32x16, want: fn(__m256i) -> __m256i, name: &str| { + let g = got.to_array(); + for half in 0..2 { + let expect = read(want(load(&a_arr, half))); + assert_eq!(&g[half * 8..half * 8 + 8], &expect[..], "{name}, half {half}"); + } + }; + rot(a.rotate_left(16), |x| _mm256_or_si256(_mm256_srli_epi32(x, 16), _mm256_slli_epi32(x, 16)), "rot16"); + rot(a.rotate_left(20), |x| _mm256_or_si256(_mm256_srli_epi32(x, 12), _mm256_slli_epi32(x, 20)), "rot12"); + rot(a.rotate_left(24), |x| _mm256_or_si256(_mm256_srli_epi32(x, 8), _mm256_slli_epi32(x, 24)), "rot8"); + rot(a.rotate_left(25), |x| _mm256_or_si256(_mm256_srli_epi32(x, 7), _mm256_slli_epi32(x, 25)), "rot7"); + } + } + + /// The same six methods pinned lane-by-lane with no intrinsics, so every + /// backend — scalar / avx512 / neon / wasm / nightly — is held to the + /// identical permutation on machines where the x86 oracle cannot run. + #[test] + fn u32x16_blake3_shuffles_are_lane_exact() { + let a: [u32; 16] = core::array::from_fn(|i| 10 + i as u32); + let b: [u32; 16] = core::array::from_fn(|i| 30 + i as u32); + let (va, vb) = (U32x16::from_array(a), U32x16::from_array(b)); + + // Per 128-bit quad q: lanes 4q..4q+4, independently in each quad. + assert_eq!( + va.interleave_lo_u32(vb).to_array(), + [10, 30, 11, 31, 14, 34, 15, 35, 18, 38, 19, 39, 22, 42, 23, 43] + ); + assert_eq!( + va.interleave_hi_u32(vb).to_array(), + [12, 32, 13, 33, 16, 36, 17, 37, 20, 40, 21, 41, 24, 44, 25, 45] + ); + assert_eq!( + va.interleave_lo_u64(vb).to_array(), + [10, 11, 30, 31, 14, 15, 34, 35, 18, 19, 38, 39, 22, 23, 42, 43] + ); + assert_eq!( + va.interleave_hi_u64(vb).to_array(), + [12, 13, 32, 33, 16, 17, 36, 37, 20, 21, 40, 41, 24, 25, 44, 45] + ); + // Per 256-bit half h: lanes 8h..8h+8. + assert_eq!( + va.concat_lo_halves(vb).to_array(), + [10, 11, 12, 13, 30, 31, 32, 33, 18, 19, 20, 21, 38, 39, 40, 41] + ); + assert_eq!( + va.concat_hi_halves(vb).to_array(), + [14, 15, 16, 17, 34, 35, 36, 37, 22, 23, 24, 25, 42, 43, 44, 45] + ); + } + /// `U32x8`'s shuffle + rotate surface, checked against the REAL x86 /// intrinsics it claims to reproduce. /// diff --git a/src/simd_avx2.rs b/src/simd_avx2.rs index 8a5067d3..b805eee6 100644 --- a/src/simd_avx2.rs +++ b/src/simd_avx2.rs @@ -1542,6 +1542,133 @@ avx2_int_type!(U16x32, u16, 32, 0u16); avx2_int_type!(U32x16, u32, 16, 0u32); avx2_int_type!(U64x8, u64, 8, 0u64); +/// BLAKE3 `hash_many` shuffle surface — the transpose network's four unpacks +/// and two half-concatenations, at DEGREE 16. +/// +/// **Why these live on `U32x16` and not on a half-width type.** BLAKE3's +/// AVX2 backend is DEGREE 8 over `__m256i`. Two of those fit in one +/// `U32x16`, and every operation in `hash_many` is either lane-wise +/// (add / xor / rotate) or confined *within* a 128- or 256-bit lane — which +/// is exactly what the methods below preserve. The two 8-lane groups +/// therefore never interact, and the algorithm runs on both at once as +/// DEGREE 16 with no cross-talk. A `U32x8` is neither needed nor wanted +/// (operator ruling, 2026-07-28: a half-width type standing in for the lane +/// the substrate actually uses is an absolute no-go). +/// +/// Semantics are x86's, applied to **each 256-bit half independently**: +/// `interleave_*_u32` / `interleave_*_u64` reproduce +/// `_mm256_unpack{lo,hi}_epi{32,64}` within each 128-bit quad, and +/// `concat_{lo,hi}_halves` reproduce +/// `_mm256_permute2x128_si256(_, _, 0x20 / 0x31)` within each 256-bit half. +/// The per-lane structure is load-bearing: BLAKE3's transpose is defined in +/// terms of it, so a "helpful" whole-vector interleave would compute a +/// different permutation and produce wrong hashes with no compile error. +/// +/// Every body is a plain index loop. The codegen oracle +/// (`.claude/knowledge/simd-codegen-oracle/`) measured this exact shape: +/// fixed two-source permutations written as index loops compile to real +/// packed shuffles, and a transpose composed from them emits +/// `vpunpcklqdq` / `vpermq` / `vinserti128`. No `unsafe`, no `core::arch`, +/// no intrinsic override earned. See +/// `.claude/knowledge/blake3-on-ndarray-simd.md`. +impl U32x16 { + /// `_mm256_unpacklo_epi32` per 256-bit half: within each 128-bit quad, + /// interleave the low two `u32` of each operand. + #[inline(always)] + pub fn interleave_lo_u32(self, other: Self) -> Self { + let (a, b) = (self.to_array(), other.to_array()); + let mut o = [0u32; 16]; + for q in 0..4 { + let i = 4 * q; + o[i] = a[i]; + o[i + 1] = b[i]; + o[i + 2] = a[i + 1]; + o[i + 3] = b[i + 1]; + } + Self::from_array(o) + } + + /// `_mm256_unpackhi_epi32` per 256-bit half: within each 128-bit quad, + /// interleave the high two `u32` of each operand. + #[inline(always)] + pub fn interleave_hi_u32(self, other: Self) -> Self { + let (a, b) = (self.to_array(), other.to_array()); + let mut o = [0u32; 16]; + for q in 0..4 { + let i = 4 * q; + o[i] = a[i + 2]; + o[i + 1] = b[i + 2]; + o[i + 2] = a[i + 3]; + o[i + 3] = b[i + 3]; + } + Self::from_array(o) + } + + /// `_mm256_unpacklo_epi64` per 256-bit half: within each 128-bit quad, + /// the low `u64` of each operand. + #[inline(always)] + pub fn interleave_lo_u64(self, other: Self) -> Self { + let (a, b) = (self.to_array(), other.to_array()); + let mut o = [0u32; 16]; + for q in 0..4 { + let i = 4 * q; + o[i] = a[i]; + o[i + 1] = a[i + 1]; + o[i + 2] = b[i]; + o[i + 3] = b[i + 1]; + } + Self::from_array(o) + } + + /// `_mm256_unpackhi_epi64` per 256-bit half: within each 128-bit quad, + /// the high `u64` of each operand. + #[inline(always)] + pub fn interleave_hi_u64(self, other: Self) -> Self { + let (a, b) = (self.to_array(), other.to_array()); + let mut o = [0u32; 16]; + for q in 0..4 { + let i = 4 * q; + o[i] = a[i + 2]; + o[i + 1] = a[i + 3]; + o[i + 2] = b[i + 2]; + o[i + 3] = b[i + 3]; + } + Self::from_array(o) + } + + /// `_mm256_permute2x128_si256(a, b, 0x20)` per 256-bit half: the low + /// 128-bit lane of each operand, concatenated. + /// + /// Only the two immediates BLAKE3's transpose uses are exposed, as named + /// methods rather than a generic `const IMM` permute: the remaining + /// immediates would have no caller and no parity test. + #[inline(always)] + pub fn concat_lo_halves(self, other: Self) -> Self { + let (a, b) = (self.to_array(), other.to_array()); + let mut o = [0u32; 16]; + for h in 0..2 { + let i = 8 * h; + o[i..i + 4].copy_from_slice(&a[i..i + 4]); + o[i + 4..i + 8].copy_from_slice(&b[i..i + 4]); + } + Self::from_array(o) + } + + /// `_mm256_permute2x128_si256(a, b, 0x31)` per 256-bit half: the high + /// 128-bit lane of each operand, concatenated. + #[inline(always)] + pub fn concat_hi_halves(self, other: Self) -> Self { + let (a, b) = (self.to_array(), other.to_array()); + let mut o = [0u32; 16]; + for h in 0..2 { + let i = 8 * h; + o[i..i + 4].copy_from_slice(&a[i + 4..i + 8]); + o[i + 4..i + 8].copy_from_slice(&b[i + 4..i + 8]); + } + Self::from_array(o) + } +} + impl U32x16 { /// Lane-wise left-rotate by `n` bits — the ARX rotate (matches /// `u32::rotate_left`), completing `Add` + `BitXor` for ChaCha20/BLAKE. diff --git a/src/simd_avx512.rs b/src/simd_avx512.rs index 54021524..c9eba078 100644 --- a/src/simd_avx512.rs +++ b/src/simd_avx512.rs @@ -1396,6 +1396,133 @@ impl F32x8 { #[repr(transparent)] pub struct U32x16(pub __m512i); +/// BLAKE3 `hash_many` shuffle surface — the transpose network's four unpacks +/// and two half-concatenations, at DEGREE 16. +/// +/// **Why these live on `U32x16` and not on a half-width type.** BLAKE3's +/// AVX2 backend is DEGREE 8 over `__m256i`. Two of those fit in one +/// `U32x16`, and every operation in `hash_many` is either lane-wise +/// (add / xor / rotate) or confined *within* a 128- or 256-bit lane — which +/// is exactly what the methods below preserve. The two 8-lane groups +/// therefore never interact, and the algorithm runs on both at once as +/// DEGREE 16 with no cross-talk. A `U32x8` is neither needed nor wanted +/// (operator ruling, 2026-07-28: a half-width type standing in for the lane +/// the substrate actually uses is an absolute no-go). +/// +/// Semantics are x86's, applied to **each 256-bit half independently**: +/// `interleave_*_u32` / `interleave_*_u64` reproduce +/// `_mm256_unpack{lo,hi}_epi{32,64}` within each 128-bit quad, and +/// `concat_{lo,hi}_halves` reproduce +/// `_mm256_permute2x128_si256(_, _, 0x20 / 0x31)` within each 256-bit half. +/// The per-lane structure is load-bearing: BLAKE3's transpose is defined in +/// terms of it, so a "helpful" whole-vector interleave would compute a +/// different permutation and produce wrong hashes with no compile error. +/// +/// Every body is a plain index loop. The codegen oracle +/// (`.claude/knowledge/simd-codegen-oracle/`) measured this exact shape: +/// fixed two-source permutations written as index loops compile to real +/// packed shuffles, and a transpose composed from them emits +/// `vpunpcklqdq` / `vpermq` / `vinserti128`. No `unsafe`, no `core::arch`, +/// no intrinsic override earned. See +/// `.claude/knowledge/blake3-on-ndarray-simd.md`. +impl U32x16 { + /// `_mm256_unpacklo_epi32` per 256-bit half: within each 128-bit quad, + /// interleave the low two `u32` of each operand. + #[inline(always)] + pub fn interleave_lo_u32(self, other: Self) -> Self { + let (a, b) = (self.to_array(), other.to_array()); + let mut o = [0u32; 16]; + for q in 0..4 { + let i = 4 * q; + o[i] = a[i]; + o[i + 1] = b[i]; + o[i + 2] = a[i + 1]; + o[i + 3] = b[i + 1]; + } + Self::from_array(o) + } + + /// `_mm256_unpackhi_epi32` per 256-bit half: within each 128-bit quad, + /// interleave the high two `u32` of each operand. + #[inline(always)] + pub fn interleave_hi_u32(self, other: Self) -> Self { + let (a, b) = (self.to_array(), other.to_array()); + let mut o = [0u32; 16]; + for q in 0..4 { + let i = 4 * q; + o[i] = a[i + 2]; + o[i + 1] = b[i + 2]; + o[i + 2] = a[i + 3]; + o[i + 3] = b[i + 3]; + } + Self::from_array(o) + } + + /// `_mm256_unpacklo_epi64` per 256-bit half: within each 128-bit quad, + /// the low `u64` of each operand. + #[inline(always)] + pub fn interleave_lo_u64(self, other: Self) -> Self { + let (a, b) = (self.to_array(), other.to_array()); + let mut o = [0u32; 16]; + for q in 0..4 { + let i = 4 * q; + o[i] = a[i]; + o[i + 1] = a[i + 1]; + o[i + 2] = b[i]; + o[i + 3] = b[i + 1]; + } + Self::from_array(o) + } + + /// `_mm256_unpackhi_epi64` per 256-bit half: within each 128-bit quad, + /// the high `u64` of each operand. + #[inline(always)] + pub fn interleave_hi_u64(self, other: Self) -> Self { + let (a, b) = (self.to_array(), other.to_array()); + let mut o = [0u32; 16]; + for q in 0..4 { + let i = 4 * q; + o[i] = a[i + 2]; + o[i + 1] = a[i + 3]; + o[i + 2] = b[i + 2]; + o[i + 3] = b[i + 3]; + } + Self::from_array(o) + } + + /// `_mm256_permute2x128_si256(a, b, 0x20)` per 256-bit half: the low + /// 128-bit lane of each operand, concatenated. + /// + /// Only the two immediates BLAKE3's transpose uses are exposed, as named + /// methods rather than a generic `const IMM` permute: the remaining + /// immediates would have no caller and no parity test. + #[inline(always)] + pub fn concat_lo_halves(self, other: Self) -> Self { + let (a, b) = (self.to_array(), other.to_array()); + let mut o = [0u32; 16]; + for h in 0..2 { + let i = 8 * h; + o[i..i + 4].copy_from_slice(&a[i..i + 4]); + o[i + 4..i + 8].copy_from_slice(&b[i..i + 4]); + } + Self::from_array(o) + } + + /// `_mm256_permute2x128_si256(a, b, 0x31)` per 256-bit half: the high + /// 128-bit lane of each operand, concatenated. + #[inline(always)] + pub fn concat_hi_halves(self, other: Self) -> Self { + let (a, b) = (self.to_array(), other.to_array()); + let mut o = [0u32; 16]; + for h in 0..2 { + let i = 8 * h; + o[i..i + 4].copy_from_slice(&a[i + 4..i + 8]); + o[i + 4..i + 8].copy_from_slice(&b[i + 4..i + 8]); + } + Self::from_array(o) + } +} + impl U32x16 { pub const LANES: usize = 16; diff --git a/src/simd_neon.rs b/src/simd_neon.rs index 625e0112..5c66b4e9 100644 --- a/src/simd_neon.rs +++ b/src/simd_neon.rs @@ -1600,6 +1600,134 @@ impl U32x4 { #[repr(align(64))] pub struct U32x16(pub [U32x4; 4]); +#[cfg(target_arch = "aarch64")] +/// BLAKE3 `hash_many` shuffle surface — the transpose network's four unpacks +/// and two half-concatenations, at DEGREE 16. +/// +/// **Why these live on `U32x16` and not on a half-width type.** BLAKE3's +/// AVX2 backend is DEGREE 8 over `__m256i`. Two of those fit in one +/// `U32x16`, and every operation in `hash_many` is either lane-wise +/// (add / xor / rotate) or confined *within* a 128- or 256-bit lane — which +/// is exactly what the methods below preserve. The two 8-lane groups +/// therefore never interact, and the algorithm runs on both at once as +/// DEGREE 16 with no cross-talk. A `U32x8` is neither needed nor wanted +/// (operator ruling, 2026-07-28: a half-width type standing in for the lane +/// the substrate actually uses is an absolute no-go). +/// +/// Semantics are x86's, applied to **each 256-bit half independently**: +/// `interleave_*_u32` / `interleave_*_u64` reproduce +/// `_mm256_unpack{lo,hi}_epi{32,64}` within each 128-bit quad, and +/// `concat_{lo,hi}_halves` reproduce +/// `_mm256_permute2x128_si256(_, _, 0x20 / 0x31)` within each 256-bit half. +/// The per-lane structure is load-bearing: BLAKE3's transpose is defined in +/// terms of it, so a "helpful" whole-vector interleave would compute a +/// different permutation and produce wrong hashes with no compile error. +/// +/// Every body is a plain index loop. The codegen oracle +/// (`.claude/knowledge/simd-codegen-oracle/`) measured this exact shape: +/// fixed two-source permutations written as index loops compile to real +/// packed shuffles, and a transpose composed from them emits +/// `vpunpcklqdq` / `vpermq` / `vinserti128`. No `unsafe`, no `core::arch`, +/// no intrinsic override earned. See +/// `.claude/knowledge/blake3-on-ndarray-simd.md`. +impl U32x16 { + /// `_mm256_unpacklo_epi32` per 256-bit half: within each 128-bit quad, + /// interleave the low two `u32` of each operand. + #[inline(always)] + pub fn interleave_lo_u32(self, other: Self) -> Self { + let (a, b) = (self.to_array(), other.to_array()); + let mut o = [0u32; 16]; + for q in 0..4 { + let i = 4 * q; + o[i] = a[i]; + o[i + 1] = b[i]; + o[i + 2] = a[i + 1]; + o[i + 3] = b[i + 1]; + } + Self::from_array(o) + } + + /// `_mm256_unpackhi_epi32` per 256-bit half: within each 128-bit quad, + /// interleave the high two `u32` of each operand. + #[inline(always)] + pub fn interleave_hi_u32(self, other: Self) -> Self { + let (a, b) = (self.to_array(), other.to_array()); + let mut o = [0u32; 16]; + for q in 0..4 { + let i = 4 * q; + o[i] = a[i + 2]; + o[i + 1] = b[i + 2]; + o[i + 2] = a[i + 3]; + o[i + 3] = b[i + 3]; + } + Self::from_array(o) + } + + /// `_mm256_unpacklo_epi64` per 256-bit half: within each 128-bit quad, + /// the low `u64` of each operand. + #[inline(always)] + pub fn interleave_lo_u64(self, other: Self) -> Self { + let (a, b) = (self.to_array(), other.to_array()); + let mut o = [0u32; 16]; + for q in 0..4 { + let i = 4 * q; + o[i] = a[i]; + o[i + 1] = a[i + 1]; + o[i + 2] = b[i]; + o[i + 3] = b[i + 1]; + } + Self::from_array(o) + } + + /// `_mm256_unpackhi_epi64` per 256-bit half: within each 128-bit quad, + /// the high `u64` of each operand. + #[inline(always)] + pub fn interleave_hi_u64(self, other: Self) -> Self { + let (a, b) = (self.to_array(), other.to_array()); + let mut o = [0u32; 16]; + for q in 0..4 { + let i = 4 * q; + o[i] = a[i + 2]; + o[i + 1] = a[i + 3]; + o[i + 2] = b[i + 2]; + o[i + 3] = b[i + 3]; + } + Self::from_array(o) + } + + /// `_mm256_permute2x128_si256(a, b, 0x20)` per 256-bit half: the low + /// 128-bit lane of each operand, concatenated. + /// + /// Only the two immediates BLAKE3's transpose uses are exposed, as named + /// methods rather than a generic `const IMM` permute: the remaining + /// immediates would have no caller and no parity test. + #[inline(always)] + pub fn concat_lo_halves(self, other: Self) -> Self { + let (a, b) = (self.to_array(), other.to_array()); + let mut o = [0u32; 16]; + for h in 0..2 { + let i = 8 * h; + o[i..i + 4].copy_from_slice(&a[i..i + 4]); + o[i + 4..i + 8].copy_from_slice(&b[i..i + 4]); + } + Self::from_array(o) + } + + /// `_mm256_permute2x128_si256(a, b, 0x31)` per 256-bit half: the high + /// 128-bit lane of each operand, concatenated. + #[inline(always)] + pub fn concat_hi_halves(self, other: Self) -> Self { + let (a, b) = (self.to_array(), other.to_array()); + let mut o = [0u32; 16]; + for h in 0..2 { + let i = 8 * h; + o[i..i + 4].copy_from_slice(&a[i + 4..i + 8]); + o[i + 4..i + 8].copy_from_slice(&b[i + 4..i + 8]); + } + Self::from_array(o) + } +} + #[cfg(target_arch = "aarch64")] impl U32x16 { pub const LANES: usize = 16; diff --git a/src/simd_nightly/u_word_types.rs b/src/simd_nightly/u_word_types.rs index b8add767..9592102e 100644 --- a/src/simd_nightly/u_word_types.rs +++ b/src/simd_nightly/u_word_types.rs @@ -292,6 +292,133 @@ impl Default for U32x8 { #[repr(transparent)] pub struct U32x16(pub u32x16); +/// BLAKE3 `hash_many` shuffle surface — the transpose network's four unpacks +/// and two half-concatenations, at DEGREE 16. +/// +/// **Why these live on `U32x16` and not on a half-width type.** BLAKE3's +/// AVX2 backend is DEGREE 8 over `__m256i`. Two of those fit in one +/// `U32x16`, and every operation in `hash_many` is either lane-wise +/// (add / xor / rotate) or confined *within* a 128- or 256-bit lane — which +/// is exactly what the methods below preserve. The two 8-lane groups +/// therefore never interact, and the algorithm runs on both at once as +/// DEGREE 16 with no cross-talk. A `U32x8` is neither needed nor wanted +/// (operator ruling, 2026-07-28: a half-width type standing in for the lane +/// the substrate actually uses is an absolute no-go). +/// +/// Semantics are x86's, applied to **each 256-bit half independently**: +/// `interleave_*_u32` / `interleave_*_u64` reproduce +/// `_mm256_unpack{lo,hi}_epi{32,64}` within each 128-bit quad, and +/// `concat_{lo,hi}_halves` reproduce +/// `_mm256_permute2x128_si256(_, _, 0x20 / 0x31)` within each 256-bit half. +/// The per-lane structure is load-bearing: BLAKE3's transpose is defined in +/// terms of it, so a "helpful" whole-vector interleave would compute a +/// different permutation and produce wrong hashes with no compile error. +/// +/// Every body is a plain index loop. The codegen oracle +/// (`.claude/knowledge/simd-codegen-oracle/`) measured this exact shape: +/// fixed two-source permutations written as index loops compile to real +/// packed shuffles, and a transpose composed from them emits +/// `vpunpcklqdq` / `vpermq` / `vinserti128`. No `unsafe`, no `core::arch`, +/// no intrinsic override earned. See +/// `.claude/knowledge/blake3-on-ndarray-simd.md`. +impl U32x16 { + /// `_mm256_unpacklo_epi32` per 256-bit half: within each 128-bit quad, + /// interleave the low two `u32` of each operand. + #[inline(always)] + pub fn interleave_lo_u32(self, other: Self) -> Self { + let (a, b) = (self.to_array(), other.to_array()); + let mut o = [0u32; 16]; + for q in 0..4 { + let i = 4 * q; + o[i] = a[i]; + o[i + 1] = b[i]; + o[i + 2] = a[i + 1]; + o[i + 3] = b[i + 1]; + } + Self::from_array(o) + } + + /// `_mm256_unpackhi_epi32` per 256-bit half: within each 128-bit quad, + /// interleave the high two `u32` of each operand. + #[inline(always)] + pub fn interleave_hi_u32(self, other: Self) -> Self { + let (a, b) = (self.to_array(), other.to_array()); + let mut o = [0u32; 16]; + for q in 0..4 { + let i = 4 * q; + o[i] = a[i + 2]; + o[i + 1] = b[i + 2]; + o[i + 2] = a[i + 3]; + o[i + 3] = b[i + 3]; + } + Self::from_array(o) + } + + /// `_mm256_unpacklo_epi64` per 256-bit half: within each 128-bit quad, + /// the low `u64` of each operand. + #[inline(always)] + pub fn interleave_lo_u64(self, other: Self) -> Self { + let (a, b) = (self.to_array(), other.to_array()); + let mut o = [0u32; 16]; + for q in 0..4 { + let i = 4 * q; + o[i] = a[i]; + o[i + 1] = a[i + 1]; + o[i + 2] = b[i]; + o[i + 3] = b[i + 1]; + } + Self::from_array(o) + } + + /// `_mm256_unpackhi_epi64` per 256-bit half: within each 128-bit quad, + /// the high `u64` of each operand. + #[inline(always)] + pub fn interleave_hi_u64(self, other: Self) -> Self { + let (a, b) = (self.to_array(), other.to_array()); + let mut o = [0u32; 16]; + for q in 0..4 { + let i = 4 * q; + o[i] = a[i + 2]; + o[i + 1] = a[i + 3]; + o[i + 2] = b[i + 2]; + o[i + 3] = b[i + 3]; + } + Self::from_array(o) + } + + /// `_mm256_permute2x128_si256(a, b, 0x20)` per 256-bit half: the low + /// 128-bit lane of each operand, concatenated. + /// + /// Only the two immediates BLAKE3's transpose uses are exposed, as named + /// methods rather than a generic `const IMM` permute: the remaining + /// immediates would have no caller and no parity test. + #[inline(always)] + pub fn concat_lo_halves(self, other: Self) -> Self { + let (a, b) = (self.to_array(), other.to_array()); + let mut o = [0u32; 16]; + for h in 0..2 { + let i = 8 * h; + o[i..i + 4].copy_from_slice(&a[i..i + 4]); + o[i + 4..i + 8].copy_from_slice(&b[i..i + 4]); + } + Self::from_array(o) + } + + /// `_mm256_permute2x128_si256(a, b, 0x31)` per 256-bit half: the high + /// 128-bit lane of each operand, concatenated. + #[inline(always)] + pub fn concat_hi_halves(self, other: Self) -> Self { + let (a, b) = (self.to_array(), other.to_array()); + let mut o = [0u32; 16]; + for h in 0..2 { + let i = 8 * h; + o[i..i + 4].copy_from_slice(&a[i + 4..i + 8]); + o[i + 4..i + 8].copy_from_slice(&b[i + 4..i + 8]); + } + Self::from_array(o) + } +} + impl U32x16 { pub const LANES: usize = 16; diff --git a/src/simd_scalar.rs b/src/simd_scalar.rs index 1e615e5d..2e7f09f7 100644 --- a/src/simd_scalar.rs +++ b/src/simd_scalar.rs @@ -1096,6 +1096,133 @@ impl Shl for U32x16 { } } +/// BLAKE3 `hash_many` shuffle surface — the transpose network's four unpacks +/// and two half-concatenations, at DEGREE 16. +/// +/// **Why these live on `U32x16` and not on a half-width type.** BLAKE3's +/// AVX2 backend is DEGREE 8 over `__m256i`. Two of those fit in one +/// `U32x16`, and every operation in `hash_many` is either lane-wise +/// (add / xor / rotate) or confined *within* a 128- or 256-bit lane — which +/// is exactly what the methods below preserve. The two 8-lane groups +/// therefore never interact, and the algorithm runs on both at once as +/// DEGREE 16 with no cross-talk. A `U32x8` is neither needed nor wanted +/// (operator ruling, 2026-07-28: a half-width type standing in for the lane +/// the substrate actually uses is an absolute no-go). +/// +/// Semantics are x86's, applied to **each 256-bit half independently**: +/// `interleave_*_u32` / `interleave_*_u64` reproduce +/// `_mm256_unpack{lo,hi}_epi{32,64}` within each 128-bit quad, and +/// `concat_{lo,hi}_halves` reproduce +/// `_mm256_permute2x128_si256(_, _, 0x20 / 0x31)` within each 256-bit half. +/// The per-lane structure is load-bearing: BLAKE3's transpose is defined in +/// terms of it, so a "helpful" whole-vector interleave would compute a +/// different permutation and produce wrong hashes with no compile error. +/// +/// Every body is a plain index loop. The codegen oracle +/// (`.claude/knowledge/simd-codegen-oracle/`) measured this exact shape: +/// fixed two-source permutations written as index loops compile to real +/// packed shuffles, and a transpose composed from them emits +/// `vpunpcklqdq` / `vpermq` / `vinserti128`. No `unsafe`, no `core::arch`, +/// no intrinsic override earned. See +/// `.claude/knowledge/blake3-on-ndarray-simd.md`. +impl U32x16 { + /// `_mm256_unpacklo_epi32` per 256-bit half: within each 128-bit quad, + /// interleave the low two `u32` of each operand. + #[inline(always)] + pub fn interleave_lo_u32(self, other: Self) -> Self { + let (a, b) = (self.to_array(), other.to_array()); + let mut o = [0u32; 16]; + for q in 0..4 { + let i = 4 * q; + o[i] = a[i]; + o[i + 1] = b[i]; + o[i + 2] = a[i + 1]; + o[i + 3] = b[i + 1]; + } + Self::from_array(o) + } + + /// `_mm256_unpackhi_epi32` per 256-bit half: within each 128-bit quad, + /// interleave the high two `u32` of each operand. + #[inline(always)] + pub fn interleave_hi_u32(self, other: Self) -> Self { + let (a, b) = (self.to_array(), other.to_array()); + let mut o = [0u32; 16]; + for q in 0..4 { + let i = 4 * q; + o[i] = a[i + 2]; + o[i + 1] = b[i + 2]; + o[i + 2] = a[i + 3]; + o[i + 3] = b[i + 3]; + } + Self::from_array(o) + } + + /// `_mm256_unpacklo_epi64` per 256-bit half: within each 128-bit quad, + /// the low `u64` of each operand. + #[inline(always)] + pub fn interleave_lo_u64(self, other: Self) -> Self { + let (a, b) = (self.to_array(), other.to_array()); + let mut o = [0u32; 16]; + for q in 0..4 { + let i = 4 * q; + o[i] = a[i]; + o[i + 1] = a[i + 1]; + o[i + 2] = b[i]; + o[i + 3] = b[i + 1]; + } + Self::from_array(o) + } + + /// `_mm256_unpackhi_epi64` per 256-bit half: within each 128-bit quad, + /// the high `u64` of each operand. + #[inline(always)] + pub fn interleave_hi_u64(self, other: Self) -> Self { + let (a, b) = (self.to_array(), other.to_array()); + let mut o = [0u32; 16]; + for q in 0..4 { + let i = 4 * q; + o[i] = a[i + 2]; + o[i + 1] = a[i + 3]; + o[i + 2] = b[i + 2]; + o[i + 3] = b[i + 3]; + } + Self::from_array(o) + } + + /// `_mm256_permute2x128_si256(a, b, 0x20)` per 256-bit half: the low + /// 128-bit lane of each operand, concatenated. + /// + /// Only the two immediates BLAKE3's transpose uses are exposed, as named + /// methods rather than a generic `const IMM` permute: the remaining + /// immediates would have no caller and no parity test. + #[inline(always)] + pub fn concat_lo_halves(self, other: Self) -> Self { + let (a, b) = (self.to_array(), other.to_array()); + let mut o = [0u32; 16]; + for h in 0..2 { + let i = 8 * h; + o[i..i + 4].copy_from_slice(&a[i..i + 4]); + o[i + 4..i + 8].copy_from_slice(&b[i..i + 4]); + } + Self::from_array(o) + } + + /// `_mm256_permute2x128_si256(a, b, 0x31)` per 256-bit half: the high + /// 128-bit lane of each operand, concatenated. + #[inline(always)] + pub fn concat_hi_halves(self, other: Self) -> Self { + let (a, b) = (self.to_array(), other.to_array()); + let mut o = [0u32; 16]; + for h in 0..2 { + let i = 8 * h; + o[i..i + 4].copy_from_slice(&a[i + 4..i + 8]); + o[i + 4..i + 8].copy_from_slice(&b[i + 4..i + 8]); + } + Self::from_array(o) + } +} + impl U32x16 { /// Lane-wise left-rotate by `n` bits — the ARX rotate (matches /// `u32::rotate_left`). The completeness tier: a full, correct diff --git a/src/simd_wasm.rs b/src/simd_wasm.rs index b25756f3..da1ff192 100644 --- a/src/simd_wasm.rs +++ b/src/simd_wasm.rs @@ -927,6 +927,133 @@ pub mod wasm32_simd { #[repr(align(64))] pub struct U32x16(pub [U32x4; 4]); + /// BLAKE3 `hash_many` shuffle surface — the transpose network's four unpacks + /// and two half-concatenations, at DEGREE 16. + /// + /// **Why these live on `U32x16` and not on a half-width type.** BLAKE3's + /// AVX2 backend is DEGREE 8 over `__m256i`. Two of those fit in one + /// `U32x16`, and every operation in `hash_many` is either lane-wise + /// (add / xor / rotate) or confined *within* a 128- or 256-bit lane — which + /// is exactly what the methods below preserve. The two 8-lane groups + /// therefore never interact, and the algorithm runs on both at once as + /// DEGREE 16 with no cross-talk. A `U32x8` is neither needed nor wanted + /// (operator ruling, 2026-07-28: a half-width type standing in for the lane + /// the substrate actually uses is an absolute no-go). + /// + /// Semantics are x86's, applied to **each 256-bit half independently**: + /// `interleave_*_u32` / `interleave_*_u64` reproduce + /// `_mm256_unpack{lo,hi}_epi{32,64}` within each 128-bit quad, and + /// `concat_{lo,hi}_halves` reproduce + /// `_mm256_permute2x128_si256(_, _, 0x20 / 0x31)` within each 256-bit half. + /// The per-lane structure is load-bearing: BLAKE3's transpose is defined in + /// terms of it, so a "helpful" whole-vector interleave would compute a + /// different permutation and produce wrong hashes with no compile error. + /// + /// Every body is a plain index loop. The codegen oracle + /// (`.claude/knowledge/simd-codegen-oracle/`) measured this exact shape: + /// fixed two-source permutations written as index loops compile to real + /// packed shuffles, and a transpose composed from them emits + /// `vpunpcklqdq` / `vpermq` / `vinserti128`. No `unsafe`, no `core::arch`, + /// no intrinsic override earned. See + /// `.claude/knowledge/blake3-on-ndarray-simd.md`. + impl U32x16 { + /// `_mm256_unpacklo_epi32` per 256-bit half: within each 128-bit quad, + /// interleave the low two `u32` of each operand. + #[inline(always)] + pub fn interleave_lo_u32(self, other: Self) -> Self { + let (a, b) = (self.to_array(), other.to_array()); + let mut o = [0u32; 16]; + for q in 0..4 { + let i = 4 * q; + o[i] = a[i]; + o[i + 1] = b[i]; + o[i + 2] = a[i + 1]; + o[i + 3] = b[i + 1]; + } + Self::from_array(o) + } + + /// `_mm256_unpackhi_epi32` per 256-bit half: within each 128-bit quad, + /// interleave the high two `u32` of each operand. + #[inline(always)] + pub fn interleave_hi_u32(self, other: Self) -> Self { + let (a, b) = (self.to_array(), other.to_array()); + let mut o = [0u32; 16]; + for q in 0..4 { + let i = 4 * q; + o[i] = a[i + 2]; + o[i + 1] = b[i + 2]; + o[i + 2] = a[i + 3]; + o[i + 3] = b[i + 3]; + } + Self::from_array(o) + } + + /// `_mm256_unpacklo_epi64` per 256-bit half: within each 128-bit quad, + /// the low `u64` of each operand. + #[inline(always)] + pub fn interleave_lo_u64(self, other: Self) -> Self { + let (a, b) = (self.to_array(), other.to_array()); + let mut o = [0u32; 16]; + for q in 0..4 { + let i = 4 * q; + o[i] = a[i]; + o[i + 1] = a[i + 1]; + o[i + 2] = b[i]; + o[i + 3] = b[i + 1]; + } + Self::from_array(o) + } + + /// `_mm256_unpackhi_epi64` per 256-bit half: within each 128-bit quad, + /// the high `u64` of each operand. + #[inline(always)] + pub fn interleave_hi_u64(self, other: Self) -> Self { + let (a, b) = (self.to_array(), other.to_array()); + let mut o = [0u32; 16]; + for q in 0..4 { + let i = 4 * q; + o[i] = a[i + 2]; + o[i + 1] = a[i + 3]; + o[i + 2] = b[i + 2]; + o[i + 3] = b[i + 3]; + } + Self::from_array(o) + } + + /// `_mm256_permute2x128_si256(a, b, 0x20)` per 256-bit half: the low + /// 128-bit lane of each operand, concatenated. + /// + /// Only the two immediates BLAKE3's transpose uses are exposed, as named + /// methods rather than a generic `const IMM` permute: the remaining + /// immediates would have no caller and no parity test. + #[inline(always)] + pub fn concat_lo_halves(self, other: Self) -> Self { + let (a, b) = (self.to_array(), other.to_array()); + let mut o = [0u32; 16]; + for h in 0..2 { + let i = 8 * h; + o[i..i + 4].copy_from_slice(&a[i..i + 4]); + o[i + 4..i + 8].copy_from_slice(&b[i..i + 4]); + } + Self::from_array(o) + } + + /// `_mm256_permute2x128_si256(a, b, 0x31)` per 256-bit half: the high + /// 128-bit lane of each operand, concatenated. + #[inline(always)] + pub fn concat_hi_halves(self, other: Self) -> Self { + let (a, b) = (self.to_array(), other.to_array()); + let mut o = [0u32; 16]; + for h in 0..2 { + let i = 8 * h; + o[i..i + 4].copy_from_slice(&a[i + 4..i + 8]); + o[i + 4..i + 8].copy_from_slice(&b[i + 4..i + 8]); + } + Self::from_array(o) + } + } + impl U32x16 { pub const LANES: usize = 16;