From 0a076aa701153b7afb7c13efc12f8e3a6eda1ebe Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 29 Jul 2026 16:21:29 +0000 Subject: [PATCH 1/2] =?UTF-8?q?simd:=20exchange=20on=20U32x16?= =?UTF-8?q?=20=E2=80=94=20the=20transpose=20surface,=20all=20six=20backend?= =?UTF-8?q?s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit blake3-on-ndarray-simd.md identified the gap and prescribed the shape: "what is missing is a method surface on U32x16, implemented as the scalar index loops LLVM already handles... it wants to be one const-generic operation rather than five named ones." This is that operation. exchange::(lo, hi) -> (lo', hi') pairs lane c with lane c ^ G. G = 1 is the 32-bit unpack, G = 2 the 64-bit unpack, G = 4 the 128-bit lane exchange, G = 8 the 256-bit half exchange that a 512-bit lane needs and for which no AVX2 intrinsic exists. Four stages over it -- pairing row r with row r | G -- compose a complete 16x16 transpose. Why const-generic rather than five named methods: G must be compile-time constant so every shuffle pattern is fixed. That is the precondition for LLVM to select a shuffle instead of an indexed copy, and it is the difference the oracle measured between the two spellings of the same function: transpose_16x16_u32 (monolithic index loop) 0 packed 1088 B of stack, a 256-iteration scalar copy transpose_16x16_composed (four butterfly stages) 79 packed / 0 scalar 19 real shuffles: vinserti128, vpshufd, vpblendd, vpunpcklqdq, vpermq, vunpcklpd, vpermpd No intrinsic override is earned. Under the entry criterion in simd-one-spec-design.md an override needs a probe showing the generic form fails; it does not fail here. Bodies are scalar index loops, zero unsafe, zero core::arch, on every backend. Semantics note: this is NOT interleave_*/concat_*, which were added in #267 and mirror x86's per-128-bit-lane unpack. exchange pairs across the whole vector. Neither substitutes for the other, and both are now tested. Two tests. u32x16_exchange_stages_compose_a_transpose composes the four stages and checks the result against the expected transpose -- the same control the oracle driver applies to its probe-local twin, brought into the suite so the library method is verified rather than its copy. A packed-but-wrong network passes a codegen histogram and fails this. u32x16_exchange_is_lane_exact_per_ granularity pins each G independently so one bad stage cannot hide inside the composition. 34 simd tests pass on 1.95.0. Not claimed: throughput. The oracle measures instruction class, not speed -- "emits packed shuffles" is not "beats the intrinsic backend", and no benchmark against rust_avx2.rs has been run on either side. Claude-Session: https://claude.ai/code/session_01VdfbkUCBbtZhy3yjSfCDHp --- src/simd.rs | 65 ++++++++++++++++++++++++++++++++ src/simd_avx2.rs | 32 ++++++++++++++++ src/simd_avx512.rs | 32 ++++++++++++++++ src/simd_neon.rs | 32 ++++++++++++++++ src/simd_nightly/u_word_types.rs | 32 ++++++++++++++++ src/simd_scalar.rs | 32 ++++++++++++++++ src/simd_wasm.rs | 32 ++++++++++++++++ 7 files changed, 257 insertions(+) diff --git a/src/simd.rs b/src/simd.rs index 554a0ca9..888974be 100644 --- a/src/simd.rs +++ b/src/simd.rs @@ -898,6 +898,71 @@ mod tests { ); } + /// `exchange::` composes a complete 16x16 transpose over four stages. + /// + /// This is the control the codegen oracle's driver applies to + /// `transpose_16x16_composed`, brought into the test suite so the library + /// method is checked and not merely its probe-local twin. A packed-but- + /// wrong shuffle network would pass a codegen histogram and fail here. + /// + /// Note the semantics differ from `interleave_*` / `concat_*` above: those + /// mirror x86's per-128-bit-lane `unpack`, whereas `exchange` pairs lane + /// `c` with lane `c ^ G` across the whole vector. They are different + /// permutations and neither substitutes for the other. + #[test] + fn u32x16_exchange_stages_compose_a_transpose() { + fn stage(m: &mut [U32x16; 16]) { + for r in 0..16 { + if r & G == 0 { + let (lo, hi) = m[r].exchange::(m[r | G]); + m[r] = lo; + m[r | G] = hi; + } + } + } + + // Row r, lane c = r*16 + c. A transpose must yield row r, lane c = c*16 + r. + let mut m: [U32x16; 16] = core::array::from_fn(|r| { + U32x16::from_array(core::array::from_fn(|c| (r * 16 + c) as u32)) + }); + + stage::<1>(&mut m); + stage::<2>(&mut m); + stage::<4>(&mut m); + stage::<8>(&mut m); + + for (r, row) in m.iter().enumerate() { + let want: [u32; 16] = core::array::from_fn(|c| (c * 16 + r) as u32); + assert_eq!(row.to_array(), want, "row {r} after four exchange stages"); + } + } + + /// Each granularity in isolation, pinned lane-by-lane, so a backend that + /// gets one stage wrong is not masked by the composition above. + #[test] + fn u32x16_exchange_is_lane_exact_per_granularity() { + let a: [u32; 16] = core::array::from_fn(|i| 10 + i as u32); + let b: [u32; 16] = core::array::from_fn(|i| 30 + i as u32); + let (va, vb) = (U32x16::from_array(a), U32x16::from_array(b)); + + // lo[c] = a[c] when c & G == 0 else b[c ^ G] + // hi[c] = b[c] when c & G != 0 else a[c ^ G] + for g in [1usize, 2, 4, 8] { + let (lo, hi) = match g { + 1 => va.exchange::<1>(vb), + 2 => va.exchange::<2>(vb), + 4 => va.exchange::<4>(vb), + _ => va.exchange::<8>(vb), + }; + let want_lo: [u32; 16] = + core::array::from_fn(|c| if c & g == 0 { a[c] } else { b[c ^ g] }); + let want_hi: [u32; 16] = + core::array::from_fn(|c| if c & g != 0 { b[c] } else { a[c ^ g] }); + assert_eq!(lo.to_array(), want_lo, "exchange::<{g}> lo"); + assert_eq!(hi.to_array(), want_hi, "exchange::<{g}> hi"); + } + } + /// `U32x8`'s shuffle + rotate surface, checked against the REAL x86 /// intrinsics it claims to reproduce. /// diff --git a/src/simd_avx2.rs b/src/simd_avx2.rs index cd3b1e3a..0423f7fb 100644 --- a/src/simd_avx2.rs +++ b/src/simd_avx2.rs @@ -1715,6 +1715,38 @@ impl U32x16 { } Self::from_array(o) } + + /// One butterfly exchange at block granularity `G` elements — the general + /// form of the whole unpack / lane-exchange family, parameterized by + /// granularity instead of one method per width. + /// + /// `G = 1` is the 32-bit unpack, `G = 2` the 64-bit unpack, `G = 4` the + /// 128-bit lane exchange, and `G = 8` the 256-bit half exchange that a + /// 512-bit lane additionally needs and for which no AVX2 intrinsic exists. + /// `G` is a const parameter, so every shuffle pattern is compile-time + /// constant — the same property a hand-written intrinsic has, and the + /// precondition for LLVM to select a shuffle rather than an indexed copy. + /// + /// Four stages over this (`G` = 1, 2, 4, 8, pairing row `r` with `r | G`) + /// compose a complete 16x16 transpose. Measured as + /// `transpose_16x16_composed`: **79 packed / 0 scalar-lane-arith**, 19 of + /// them real shuffles. The same transpose written as one monolithic index + /// loop measures **0 packed** — 1088 bytes of stack and a 256-iteration + /// scalar copy. The spelling is the entire difference. See + /// `.claude/knowledge/blake3-on-ndarray-simd.md`. + /// + /// No intrinsic override is earned: the generic form does not fail. + #[inline(always)] + pub fn exchange(self, other: Self) -> (Self, Self) { + let (l, h) = (self.to_array(), other.to_array()); + let mut nl = [0u32; 16]; + let mut nh = [0u32; 16]; + for c in 0..16 { + nl[c] = if c & G == 0 { l[c] } else { h[c ^ G] }; + nh[c] = if c & G != 0 { h[c] } else { l[c ^ G] }; + } + (Self::from_array(nl), Self::from_array(nh)) + } } impl U32x16 { diff --git a/src/simd_avx512.rs b/src/simd_avx512.rs index 8411d4db..615ef5c2 100644 --- a/src/simd_avx512.rs +++ b/src/simd_avx512.rs @@ -1521,6 +1521,38 @@ impl U32x16 { } Self::from_array(o) } + + /// One butterfly exchange at block granularity `G` elements — the general + /// form of the whole unpack / lane-exchange family, parameterized by + /// granularity instead of one method per width. + /// + /// `G = 1` is the 32-bit unpack, `G = 2` the 64-bit unpack, `G = 4` the + /// 128-bit lane exchange, and `G = 8` the 256-bit half exchange that a + /// 512-bit lane additionally needs and for which no AVX2 intrinsic exists. + /// `G` is a const parameter, so every shuffle pattern is compile-time + /// constant — the same property a hand-written intrinsic has, and the + /// precondition for LLVM to select a shuffle rather than an indexed copy. + /// + /// Four stages over this (`G` = 1, 2, 4, 8, pairing row `r` with `r | G`) + /// compose a complete 16x16 transpose. Measured as + /// `transpose_16x16_composed`: **79 packed / 0 scalar-lane-arith**, 19 of + /// them real shuffles. The same transpose written as one monolithic index + /// loop measures **0 packed** — 1088 bytes of stack and a 256-iteration + /// scalar copy. The spelling is the entire difference. See + /// `.claude/knowledge/blake3-on-ndarray-simd.md`. + /// + /// No intrinsic override is earned: the generic form does not fail. + #[inline(always)] + pub fn exchange(self, other: Self) -> (Self, Self) { + let (l, h) = (self.to_array(), other.to_array()); + let mut nl = [0u32; 16]; + let mut nh = [0u32; 16]; + for c in 0..16 { + nl[c] = if c & G == 0 { l[c] } else { h[c ^ G] }; + nh[c] = if c & G != 0 { h[c] } else { l[c ^ G] }; + } + (Self::from_array(nl), Self::from_array(nh)) + } } impl U32x16 { diff --git a/src/simd_neon.rs b/src/simd_neon.rs index 5c66b4e9..d4d0cb45 100644 --- a/src/simd_neon.rs +++ b/src/simd_neon.rs @@ -1726,6 +1726,38 @@ impl U32x16 { } Self::from_array(o) } + + /// One butterfly exchange at block granularity `G` elements — the general + /// form of the whole unpack / lane-exchange family, parameterized by + /// granularity instead of one method per width. + /// + /// `G = 1` is the 32-bit unpack, `G = 2` the 64-bit unpack, `G = 4` the + /// 128-bit lane exchange, and `G = 8` the 256-bit half exchange that a + /// 512-bit lane additionally needs and for which no AVX2 intrinsic exists. + /// `G` is a const parameter, so every shuffle pattern is compile-time + /// constant — the same property a hand-written intrinsic has, and the + /// precondition for LLVM to select a shuffle rather than an indexed copy. + /// + /// Four stages over this (`G` = 1, 2, 4, 8, pairing row `r` with `r | G`) + /// compose a complete 16x16 transpose. Measured as + /// `transpose_16x16_composed`: **79 packed / 0 scalar-lane-arith**, 19 of + /// them real shuffles. The same transpose written as one monolithic index + /// loop measures **0 packed** — 1088 bytes of stack and a 256-iteration + /// scalar copy. The spelling is the entire difference. See + /// `.claude/knowledge/blake3-on-ndarray-simd.md`. + /// + /// No intrinsic override is earned: the generic form does not fail. + #[inline(always)] + pub fn exchange(self, other: Self) -> (Self, Self) { + let (l, h) = (self.to_array(), other.to_array()); + let mut nl = [0u32; 16]; + let mut nh = [0u32; 16]; + for c in 0..16 { + nl[c] = if c & G == 0 { l[c] } else { h[c ^ G] }; + nh[c] = if c & G != 0 { h[c] } else { l[c ^ G] }; + } + (Self::from_array(nl), Self::from_array(nh)) + } } #[cfg(target_arch = "aarch64")] diff --git a/src/simd_nightly/u_word_types.rs b/src/simd_nightly/u_word_types.rs index 93fede04..e4e379b9 100644 --- a/src/simd_nightly/u_word_types.rs +++ b/src/simd_nightly/u_word_types.rs @@ -459,6 +459,38 @@ impl U32x16 { } Self::from_array(o) } + + /// One butterfly exchange at block granularity `G` elements — the general + /// form of the whole unpack / lane-exchange family, parameterized by + /// granularity instead of one method per width. + /// + /// `G = 1` is the 32-bit unpack, `G = 2` the 64-bit unpack, `G = 4` the + /// 128-bit lane exchange, and `G = 8` the 256-bit half exchange that a + /// 512-bit lane additionally needs and for which no AVX2 intrinsic exists. + /// `G` is a const parameter, so every shuffle pattern is compile-time + /// constant — the same property a hand-written intrinsic has, and the + /// precondition for LLVM to select a shuffle rather than an indexed copy. + /// + /// Four stages over this (`G` = 1, 2, 4, 8, pairing row `r` with `r | G`) + /// compose a complete 16x16 transpose. Measured as + /// `transpose_16x16_composed`: **79 packed / 0 scalar-lane-arith**, 19 of + /// them real shuffles. The same transpose written as one monolithic index + /// loop measures **0 packed** — 1088 bytes of stack and a 256-iteration + /// scalar copy. The spelling is the entire difference. See + /// `.claude/knowledge/blake3-on-ndarray-simd.md`. + /// + /// No intrinsic override is earned: the generic form does not fail. + #[inline(always)] + pub fn exchange(self, other: Self) -> (Self, Self) { + let (l, h) = (self.to_array(), other.to_array()); + let mut nl = [0u32; 16]; + let mut nh = [0u32; 16]; + for c in 0..16 { + nl[c] = if c & G == 0 { l[c] } else { h[c ^ G] }; + nh[c] = if c & G != 0 { h[c] } else { l[c ^ G] }; + } + (Self::from_array(nl), Self::from_array(nh)) + } } impl U32x16 { diff --git a/src/simd_scalar.rs b/src/simd_scalar.rs index 70b0114f..d90387be 100644 --- a/src/simd_scalar.rs +++ b/src/simd_scalar.rs @@ -1269,6 +1269,38 @@ impl U32x16 { } Self::from_array(o) } + + /// One butterfly exchange at block granularity `G` elements — the general + /// form of the whole unpack / lane-exchange family, parameterized by + /// granularity instead of one method per width. + /// + /// `G = 1` is the 32-bit unpack, `G = 2` the 64-bit unpack, `G = 4` the + /// 128-bit lane exchange, and `G = 8` the 256-bit half exchange that a + /// 512-bit lane additionally needs and for which no AVX2 intrinsic exists. + /// `G` is a const parameter, so every shuffle pattern is compile-time + /// constant — the same property a hand-written intrinsic has, and the + /// precondition for LLVM to select a shuffle rather than an indexed copy. + /// + /// Four stages over this (`G` = 1, 2, 4, 8, pairing row `r` with `r | G`) + /// compose a complete 16x16 transpose. Measured as + /// `transpose_16x16_composed`: **79 packed / 0 scalar-lane-arith**, 19 of + /// them real shuffles. The same transpose written as one monolithic index + /// loop measures **0 packed** — 1088 bytes of stack and a 256-iteration + /// scalar copy. The spelling is the entire difference. See + /// `.claude/knowledge/blake3-on-ndarray-simd.md`. + /// + /// No intrinsic override is earned: the generic form does not fail. + #[inline(always)] + pub fn exchange(self, other: Self) -> (Self, Self) { + let (l, h) = (self.to_array(), other.to_array()); + let mut nl = [0u32; 16]; + let mut nh = [0u32; 16]; + for c in 0..16 { + nl[c] = if c & G == 0 { l[c] } else { h[c ^ G] }; + nh[c] = if c & G != 0 { h[c] } else { l[c ^ G] }; + } + (Self::from_array(nl), Self::from_array(nh)) + } } impl U32x16 { diff --git a/src/simd_wasm.rs b/src/simd_wasm.rs index da1ff192..c69f4447 100644 --- a/src/simd_wasm.rs +++ b/src/simd_wasm.rs @@ -1052,6 +1052,38 @@ pub mod wasm32_simd { } Self::from_array(o) } + + /// One butterfly exchange at block granularity `G` elements — the general + /// form of the whole unpack / lane-exchange family, parameterized by + /// granularity instead of one method per width. + /// + /// `G = 1` is the 32-bit unpack, `G = 2` the 64-bit unpack, `G = 4` the + /// 128-bit lane exchange, and `G = 8` the 256-bit half exchange that a + /// 512-bit lane additionally needs and for which no AVX2 intrinsic exists. + /// `G` is a const parameter, so every shuffle pattern is compile-time + /// constant — the same property a hand-written intrinsic has, and the + /// precondition for LLVM to select a shuffle rather than an indexed copy. + /// + /// Four stages over this (`G` = 1, 2, 4, 8, pairing row `r` with `r | G`) + /// compose a complete 16x16 transpose. Measured as + /// `transpose_16x16_composed`: **79 packed / 0 scalar-lane-arith**, 19 of + /// them real shuffles. The same transpose written as one monolithic index + /// loop measures **0 packed** — 1088 bytes of stack and a 256-iteration + /// scalar copy. The spelling is the entire difference. See + /// `.claude/knowledge/blake3-on-ndarray-simd.md`. + /// + /// No intrinsic override is earned: the generic form does not fail. + #[inline(always)] + pub fn exchange(self, other: Self) -> (Self, Self) { + let (l, h) = (self.to_array(), other.to_array()); + let mut nl = [0u32; 16]; + let mut nh = [0u32; 16]; + for c in 0..16 { + nl[c] = if c & G == 0 { l[c] } else { h[c ^ G] }; + nh[c] = if c & G != 0 { h[c] } else { l[c ^ G] }; + } + (Self::from_array(nl), Self::from_array(nh)) + } } impl U32x16 { From c8dc217aee2866adc5a99762c9b581db5a00f909 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 29 Jul 2026 16:24:46 +0000 Subject: [PATCH 2/2] simd: rustfmt the exchange tests format/stable failed on the previous commit. No logic change. --- src/simd.rs | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/simd.rs b/src/simd.rs index 888974be..2b8d25a8 100644 --- a/src/simd.rs +++ b/src/simd.rs @@ -922,9 +922,8 @@ mod tests { } // Row r, lane c = r*16 + c. A transpose must yield row r, lane c = c*16 + r. - let mut m: [U32x16; 16] = core::array::from_fn(|r| { - U32x16::from_array(core::array::from_fn(|c| (r * 16 + c) as u32)) - }); + let mut m: [U32x16; 16] = + core::array::from_fn(|r| U32x16::from_array(core::array::from_fn(|c| (r * 16 + c) as u32))); stage::<1>(&mut m); stage::<2>(&mut m); @@ -954,10 +953,8 @@ mod tests { 4 => va.exchange::<4>(vb), _ => va.exchange::<8>(vb), }; - let want_lo: [u32; 16] = - core::array::from_fn(|c| if c & g == 0 { a[c] } else { b[c ^ g] }); - let want_hi: [u32; 16] = - core::array::from_fn(|c| if c & g != 0 { b[c] } else { a[c ^ g] }); + let want_lo: [u32; 16] = core::array::from_fn(|c| if c & g == 0 { a[c] } else { b[c ^ g] }); + let want_hi: [u32; 16] = core::array::from_fn(|c| if c & g != 0 { b[c] } else { a[c ^ g] }); assert_eq!(lo.to_array(), want_lo, "exchange::<{g}> lo"); assert_eq!(hi.to_array(), want_hi, "exchange::<{g}> hi"); }